Merge pull request #14967 from ansible/facts-subset-and-cleanup

Facts subset and cleanup

docsite/rst/glossary.rst

@@ -71,8 +71,7 @@ Facts
 Facts are simply things that are discovered about remote nodes.  While they can be used in playbooks and templates just like variables, facts
 are things that are inferred, rather than set.  Facts are automatically discovered by Ansible when running plays by executing the internal 'setup'
 module on the remote nodes.  You never have to call the setup module explicitly, it just runs, but it can be disabled to save time if it is
-not needed.  For the convenience of users who are switching from other configuration management systems, the fact module will also pull in facts from the 'ohai' and 'facter'
-tools if they are installed, which are fact libraries from Chef and Puppet, respectively.
+not needed or you can tell ansible to collect only a subset of the full facts via the `gather_subset:` option. For the convenience of users who are switching from other configuration management systems, the fact module will also pull in facts from the 'ohai' and 'facter' tools if they are installed, which are fact libraries from Chef and Puppet, respectively. (These may also be disabled via `gather_subset:`)
 
 Filter Plugin
 +++++++++++++
@@ -399,7 +398,7 @@ An optional conditional statement attached to a task that is used to determine i
 Van Halen
 +++++++++
 
-For no particular reason, other than the fact that Michael really likes them, all Ansible releases are codenamed after Van Halen songs.  There is no preference given to David Lee Roth vs. Sammy Lee Hagar-era songs, and instrumentals are also allowed.  It is unlikely that there will ever be a Jump release, but a Van Halen III codename release is possible.  You never know.
+For no particular reason, other than the fact that Michael really likes them, all Ansible 0.x and 1.x releases are codenamed after Van Halen songs.  There is no preference given to David Lee Roth vs. Sammy Lee Hagar-era songs, and instrumentals are also allowed.  It is unlikely that there will ever be a Jump release, but a Van Halen III codename release is possible.  You never know.
 
 Vars (Variables)
 ++++++++++++++++

docsite/rst/intro_configuration.rst

@@ -353,6 +353,32 @@ This option can be useful for those wishing to save fact gathering time. Both 's
 
     gathering = smart
 
+.. versionadded:: 2.1
+
+You can specify a subset of gathered facts using the following option::
+
+    gather_subset = all
+
+:all: gather all subsets (the default)
+:network: gather network facts
+:hardware: gather hardware facts (longest facts to retrieve)
+:virtual: gather facts about virtual machines hosted on the machine
+:ohai: gather facts from ohai
+:facter: gather facts from facter
+
+You can combine them using a comma separated list (ex: network,virtual,facter)
+
+You can also disable specific subsets by prepending with a `!` like this::
+
+    # Don't gather hardware facts, facts from chef's ohai or puppet's facter
+    gather_subset = !hardware,!ohai,!facter
+
+A set of basic facts are always collected no matter which additional subsets
+are selected.  If you want to collect the minimal amount of facts, use
+`!all`::
+
+    gather_subset = !all
+
 hash_behaviour
 ==============
 
@@ -367,7 +393,7 @@ official examples repos do not use this setting::
 
 The valid values are either 'replace' (the default) or 'merge'.
 
-.. versionadded: '2.0'
+.. versionadded:: 2.0
 
 If you want to merge hashes without changing the global settings, use
 the `combine` filter described in :doc:`playbooks_filters`.
@@ -585,7 +611,7 @@ The directory will be created if it does not already exist.
 roles_path
 ==========
 
-.. versionadded: '1.4'
+.. versionadded:: 1.4
 
 The roles path indicate additional directories beyond the 'roles/' subdirectory of a playbook project to search to find Ansible
 roles.  For instance, if there was a source control repository of common roles and a different repository of playbooks, you might

examples/ansible.cfg

@@ -31,6 +31,18 @@
 # explicit - do not gather by default, must say gather_facts: True
 #gathering = implicit
 
+# by default retrieve all facts subsets
+# all - gather all subsets
+# network - gather min and network facts
+# hardware - gather hardware facts (longest facts to retrieve)
+# virtual - gather min and virtual facts
+# facter - import facts from facter
+# ohai - import facts from ohai
+# You can combine them using comma (ex: network,virtual)
+# You can negate them using ! (ex: !hardware,!facter,!ohai)
+# A minimal set of facts is always gathered.
+#gather_subset = all
+
 # additional paths to search for roles in, colon separated
 #roles_path    = /etc/ansible/roles
 

lib/ansible/constants.py

@@ -156,6 +156,7 @@ DEFAULT_PRIVATE_ROLE_VARS = get_config(p, DEFAULTS, 'private_role_vars', 'ANSIBL
 DEFAULT_JINJA2_EXTENSIONS = get_config(p, DEFAULTS, 'jinja2_extensions', 'ANSIBLE_JINJA2_EXTENSIONS', None)
 DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECUTABLE', '/bin/sh')
 DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
+DEFAULT_GATHER_SUBSET     = get_config(p, DEFAULTS, 'gather_subset', 'ANSIBLE_GATHER_SUBSET', 'all').lower()
 DEFAULT_LOG_PATH          = get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', '', ispath=True)
 DEFAULT_FORCE_HANDLERS    = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
 DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)

lib/ansible/executor/play_iterator.py

@@ -151,11 +151,20 @@ class PlayIterator:
         self._play = play
         self._blocks = []
 
+        # Default options to gather
+        gather_subset = C.DEFAULT_GATHER_SUBSET
+
+        # Retrieve subset to gather
+        if self._play.gather_subset is not None:
+            gather_subset = self._play.gather_subset
+
         setup_block = Block(play=self._play)
         setup_task = Task(block=setup_block)
         setup_task.action = 'setup'
         setup_task.tags   = ['always']
-        setup_task.args   = {}
+        setup_task.args   = {
+          'gather_subset': gather_subset,
+        }
         setup_task.set_loader(self._play._loader)
         setup_block.block = [setup_task]
 

lib/ansible/playbook/base.py

@@ -334,6 +334,9 @@ class Base:
                         if value is None:
                             value = []
                         elif not isinstance(value, list):
+                            if isinstance(value, string_types):
+                                value = value.split(',')
+                            else:
                                 value = [ value ]
                         if attribute.listof is not None:
                             for item in value:
@@ -346,8 +349,12 @@ class Base:
                     elif attribute.isa == 'set':
                         if value is None:
                             value = set()
+                        elif not isinstance(value, (list, set)):
+                            if isinstance(value, string_types):
+                                value = value.split(',')
                             else:
-                            if not isinstance(value, (list, set)):
+                                # Making a list like this handles strings of
+                                # text and bytes properly
                                 value = [ value ]
                         if not isinstance(value, set):
                             value = set(value)

lib/ansible/playbook/play.py

@@ -64,6 +64,7 @@ class Play(Base, Taggable, Become):
 
     # Connection
     _gather_facts        = FieldAttribute(isa='bool', default=None, always_post_validate=True)
+    _gather_subset       = FieldAttribute(isa='list', default=None, always_post_validate=True)
     _hosts               = FieldAttribute(isa='list', required=True, listof=string_types, always_post_validate=True)
     _name                = FieldAttribute(isa='string', default='', always_post_validate=True)
 

test/integration/Makefile

@@ -23,7 +23,7 @@ VAULT_PASSWORD_FILE = vault-password
 CONSUL_RUNNING := $(shell python consul_running.py)
 EUID := $(shell id -u -r)
 
-all: setup test_test_infra parsing test_var_precedence unicode test_templating_settings environment non_destructive destructive includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_connection
+all: setup test_test_infra parsing test_var_precedence unicode test_templating_settings environment non_destructive destructive includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_connection test_gathering_facts
 
 test_test_infra:
 	[ "$$(ansible-playbook -i $(INVENTORY) test_test_infra.yml -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS) | fgrep works | xargs)" = "msg: fail works (True) msg: assert works (True)" ]
@@ -70,6 +70,9 @@ unicode: setup
 test_templating_settings: setup
 	ansible-playbook test_templating_settings.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
+test_gathering_facts: setup
+	ansible-playbook test_gathering_facts.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v $(TEST_FLAGS)
+
 environment: setup
 	ansible-playbook test_environment.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)
 

test/integration/inventory

@@ -4,6 +4,9 @@ testhost2 ansible_ssh_host=127.0.0.1 ansible_connection=local
 # For testing delegate_to
 testhost3 ansible_ssh_host=127.0.0.3
 testhost4 ansible_ssh_host=127.0.0.4
+# For testing fact gathering
+facthost[0:7] ansible_host=1270.0.0.1 ansible_connection=local
+
 
 # the following inline declarations are accompanied
 # by (preferred) group_vars/ and host_vars/ variables

test/integration/test_gathering_facts.yml

@@ -0,0 +1,116 @@
+---
+
+- hosts: facthost0
+  tags: [ 'fact_min' ]
+  connection: local
+  gather_subset: "!all"
+  gather_facts: yes
+  tasks:
+    - name: Test that only retrieving minimal facts work
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" == "UNDEF"'
+
+- hosts: facthost1
+  tags: [ 'fact_network' ]
+  connection: local
+  gather_subset: "network"
+  gather_facts: yes
+  tasks:
+    - name: Test that retrieving network facts work
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" == "UNDEF"'
+
+- hosts: facthost2
+  tags: [ 'fact_hardware' ]
+  connection: local
+  gather_subset: "hardware"
+  gather_facts: yes
+  tasks:
+    - name: Test that retrieving hardware facts work
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" == "UNDEF"'
+
+- hosts: facthost3
+  tags: [ 'fact_virtual' ]
+  connection: local
+  gather_subset: "virtual"
+  gather_facts: yes
+  tasks:
+    - name: Test that retrieving virtualization facts work
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" != "UNDEF"'
+
+- hosts: facthost4
+  tags: [ 'fact_comma_string' ]
+  connection: local
+  gather_subset: "virtual,network"
+  gather_facts: yes
+  tasks:
+    - name: Test that retrieving virtualization and network as a string works
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" != "UNDEF"'
+
+- hosts: facthost5
+  tags: [ 'fact_yaml_list' ]
+  connection: local
+  gather_subset:
+    - virtual
+    - network
+  gather_facts: yes
+  tasks:
+    - name: Test that retrieving virtualization and network as a string works
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" != "UNDEF"'
+
+- hosts: facthost6
+  tags: [ 'fact_negation' ]
+  connection: local
+  gather_subset: "!hardware"
+  gather_facts: yes
+  tasks:
+    - name: Test that negation of fact subsets work
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" != "UNDEF"'
+
+- hosts: facthost7
+  tags: [ 'fact_mixed_negation_addition' ]
+  connection: local
+  gather_subset: "!hardware,network"
+  gather_facts: yes
+  tasks:
+    - name: Test that negation and additional subsets work together
+      assert:
+        that:
+          - '"{{ ansible_user_id|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_interfaces|default("UNDEF") }}" != "UNDEF"'
+          - '"{{ ansible_mounts|default("UNDEF") }}" == "UNDEF"'
+          - '"{{ ansible_virtualization_role|default("UNDEF") }}" == "UNDEF"'
+