feat(lookup/merge_variables): Add all hosts mode to collect configuration across multiple hosts (#7999)

* Add Feature to collect variables accross different hosts

* fix merging lists

* adjust unit tests

* lint fixes

* adjusting integration tests

* remove white spaces

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* apply suggested changes to correctly handling the initial_value parameter, incl. additional test

* whitespace

---------

Co-authored-by: Alexander Petrenz <alexander.petrenz@posteo.de>
Co-authored-by: Felix Fontein <felix@fontein.de>

.github/BOTMETA.yml

@@ -265,7 +265,7 @@ files:
     labels: manifold
     maintainers: galanoff
   $lookups/merge_variables.py:
-    maintainers: rlenferink m-a-r-k-e
+    maintainers: rlenferink m-a-r-k-e alpex8
   $lookups/onepass:
     labels: onepassword
     maintainers: samdoran

plugins/lookup/merge_variables.py

@@ -10,6 +10,7 @@ DOCUMENTATION = """
     author:
       - Roy Lenferink (@rlenferink)
       - Mark Ettema (@m-a-r-k-e)
+      - Alexander Petrenz (@alpex8)
     name: merge_variables
     short_description: merge variables with a certain suffix
     description:
@@ -61,6 +62,13 @@ DOCUMENTATION = """
         ini:
           - section: merge_variables_lookup
             key: override
+      groups:
+        description:
+          - Search for variables accross hosts that belong to the given groups. This allows to collect configuration pieces
+            accross different hosts (for example a service on a host with its database on another host).
+        type: list
+        elements: str
+        version_added: 8.5.0
 """
 
 EXAMPLES = """
@@ -131,22 +139,39 @@ def _verify_and_get_type(variable):
 
 
 class LookupModule(LookupBase):
-
     def run(self, terms, variables=None, **kwargs):
         self.set_options(direct=kwargs)
         initial_value = self.get_option("initial_value", None)
         self._override = self.get_option('override', 'error')
         self._pattern_type = self.get_option('pattern_type', 'regex')
+        self._groups = self.get_option('groups', None)
 
         ret = []
         for term in terms:
             if not isinstance(term, str):
                 raise AnsibleError("Non-string type '{0}' passed, only 'str' types are allowed!".format(type(term)))
 
+            if not self._groups:  # consider only own variables
                 ret.append(self._merge_vars(term, initial_value, variables))
+            else:  # consider variables of hosts in given groups
+                cross_host_merge_result = initial_value
+                for host in variables["hostvars"]:
+                    if self._is_host_in_allowed_groups(variables["hostvars"][host]["group_names"]):
+                        cross_host_merge_result = self._merge_vars(term, cross_host_merge_result, variables["hostvars"][host])
+                ret.append(cross_host_merge_result)
 
         return ret
 
+    def _is_host_in_allowed_groups(self, host_groups):
+        if 'all' in self._groups:
+            return True
+
+        group_intersection = [host_group_name for host_group_name in host_groups if host_group_name in self._groups]
+        if group_intersection:
+            return True
+
+        return False
+
     def _var_matches(self, key, search_pattern):
         if self._pattern_type == "prefix":
             return key.startswith(search_pattern)
@@ -162,7 +187,6 @@ class LookupModule(LookupBase):
         display.vvv("Merge variables with {0}: {1}".format(self._pattern_type, search_pattern))
         var_merge_names = sorted([key for key in variables.keys() if self._var_matches(key, search_pattern)])
         display.vvv("The following variables will be merged: {0}".format(var_merge_names))
-
         prev_var_type = None
         result = None
 

tests/integration/targets/lookup_merge_variables/runme.sh

@@ -11,3 +11,6 @@ ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
 ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
 ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \
     ansible-playbook test_with_env.yml "$@"
+
+ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
+    ansible-playbook -i test_inventory_all_hosts.yml test_all_hosts.yml "$@"

tests/integration/targets/lookup_merge_variables/test_all_hosts.yml

@@ -0,0 +1,64 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test merge_variables lookup plugin (multiple hosts)
+  hosts: host0
+  gather_facts: false
+  tasks:
+    - name: Test merge dicts via all group
+      delegate_to: localhost
+      vars:
+        merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_ex', pattern_type='suffix', groups=['all']) }}"
+      block:
+        - name: Test merge dicts via all group - Print the merged dict
+          ansible.builtin.debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Test merge dicts via all group - Validate that the dict is complete
+          ansible.builtin.assert:
+            that:
+              - "(merged_dict.keys() | list | length) == 4"
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'item3' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "merged_dict.list_item | length == 3"
+    - name: Test merge dicts via two of three groups
+      delegate_to: localhost
+      vars:
+        merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', pattern_type='suffix', groups=['dummy1', 'dummy2']) }}"
+      block:
+        - name: Test merge dicts via two of three groups - Print the merged dict
+          ansible.builtin.debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Test merge dicts via two of three groups - Validate that the dict is complete
+          ansible.builtin.assert:
+            that:
+              - "(merged_dict.keys() | list | length) == 3"
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "merged_dict.list_item | length == 2"
+    - name: Test merge dicts via two of three groups with inital value
+      delegate_to: localhost
+      vars:
+        initial_dict:
+          initial: initial_value
+        merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', initial_value=initial_dict, pattern_type='suffix', groups=['dummy1', 'dummy2']) }}"
+      block:
+        - name: Test merge dicts via two of three groups with inital value - Print the merged dict
+          ansible.builtin.debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Test merge dicts via two of three groups with inital value - Validate that the dict is complete
+          ansible.builtin.assert:
+            that:
+              - "(merged_dict.keys() | list | length) == 4"
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "merged_dict.list_item | length == 2"

tests/integration/targets/lookup_merge_variables/test_inventory_all_hosts.yml

@@ -0,0 +1,52 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+all:
+  hosts:
+    host0:
+    host1:
+      testdict1__merge_dict_ex:
+        item1: value1
+        list_item:
+          - test1
+
+      testdict2__merge_dict_ex:
+        item2: value2
+        list_item:
+          - test2
+
+      testdict__merge_dict_in:
+        item1: value1
+        list_item:
+          - test1
+    host2:
+      testdict3__merge_dict_ex:
+        item3: value3
+        list_item:
+          - test3
+
+      testdict__merge_dict_in:
+        item2: value2
+        list_item:
+          - test2
+
+    host3:
+      testdict__merge_dict_in:
+        item3: value3
+        list_item:
+          - test3
+
+dummy1:
+  hosts:
+    host1:
+
+dummy2:
+  hosts:
+    host2:
+
+dummy3:
+  hosts:
+    host3:

tests/unit/plugins/lookup/test_merge_variables.py

@@ -24,7 +24,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
         self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar)
 
     @patch.object(AnsiblePlugin, 'set_options')
-    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
     def test_merge_list(self, mock_set_options, mock_get_option, mock_template):
         results = self.merge_vars_lookup.run(['__merge_list'], {
@@ -36,7 +36,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
         self.assertEqual(results, [['item1', 'item3']])
 
     @patch.object(AnsiblePlugin, 'set_options')
-    @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix'])
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
     def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
         results = self.merge_vars_lookup.run(['__merge_list'], {
@@ -48,7 +48,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
         self.assertEqual(results, [['initial_item', 'item1', 'item3']])
 
     @patch.object(AnsiblePlugin, 'set_options')
-    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
                                                     {'item2': 'test', 'list_item': ['test2']}])
     def test_merge_dict(self, mock_set_options, mock_get_option, mock_template):
@@ -73,7 +73,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
 
     @patch.object(AnsiblePlugin, 'set_options')
     @patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']},
-                                                            'ignore', 'suffix'])
+                                                            'ignore', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
                                                     {'item2': 'test', 'list_item': ['test2']}])
     def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
@@ -98,7 +98,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
         ])
 
     @patch.object(AnsiblePlugin, 'set_options')
-    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix'])
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
     @patch.object(Display, 'warning')
     def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display):
@@ -111,7 +111,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
         self.assertEqual(results, [{'item': 'value2'}])
 
     @patch.object(AnsiblePlugin, 'set_options')
-    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix'])
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
     def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template):
         with self.assertRaises(AnsibleError):
@@ -121,7 +121,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
             })
 
     @patch.object(AnsiblePlugin, 'set_options')
-    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None])
     @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
                                                     ['item2', 'item3']])
     def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template):
@@ -133,3 +133,150 @@ class TestMergeVariablesLookup(unittest.TestCase):
                 },
                 'testdict__merge_var': ['item2', 'item3']
             })
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['all']])
+    @patch.object(Templar, 'template', side_effect=[
+        {'var': [{'item1': 'value1', 'item2': 'value2'}]},
+        {'var': [{'item5': 'value5', 'item6': 'value6'}]},
+    ])
+    def test_merge_dict_group_all(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_var'], {
+            'inventory_hostname': 'host1',
+            'hostvars': {
+                'host1': {
+                    'group_names': ['dummy1'],
+                    'inventory_hostname': 'host1',
+                    '1testlist__merge_var': {
+                        'var': [{'item1': 'value1', 'item2': 'value2'}]
+                    }
+                },
+                'host2': {
+                    'group_names': ['dummy1'],
+                    'inventory_hostname': 'host2',
+                    '2otherlist__merge_var': {
+                        'var': [{'item5': 'value5', 'item6': 'value6'}]
+                    }
+                }
+            }
+        })
+
+        self.assertEqual(results, [
+            {'var': [
+                {'item1': 'value1', 'item2': 'value2'},
+                {'item5': 'value5', 'item6': 'value6'}
+            ]}
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1']])
+    @patch.object(Templar, 'template', side_effect=[
+        {'var': [{'item1': 'value1', 'item2': 'value2'}]},
+        {'var': [{'item5': 'value5', 'item6': 'value6'}]},
+    ])
+    def test_merge_dict_group_single(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_var'], {
+            'inventory_hostname': 'host1',
+            'hostvars': {
+                'host1': {
+                    'group_names': ['dummy1'],
+                    'inventory_hostname': 'host1',
+                    '1testlist__merge_var': {
+                        'var': [{'item1': 'value1', 'item2': 'value2'}]
+                    }
+                },
+                'host2': {
+                    'group_names': ['dummy1'],
+                    'inventory_hostname': 'host2',
+                    '2otherlist__merge_var': {
+                        'var': [{'item5': 'value5', 'item6': 'value6'}]
+                    }
+                },
+                'host3': {
+                    'group_names': ['dummy2'],
+                    'inventory_hostname': 'host3',
+                    '3otherlist__merge_var': {
+                        'var': [{'item3': 'value3', 'item4': 'value4'}]
+                    }
+                }
+            }
+        })
+
+        self.assertEqual(results, [
+            {'var': [
+                {'item1': 'value1', 'item2': 'value2'},
+                {'item5': 'value5', 'item6': 'value6'}
+            ]}
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1', 'dummy2']])
+    @patch.object(Templar, 'template', side_effect=[
+        {'var': [{'item1': 'value1', 'item2': 'value2'}]},
+        {'var': [{'item5': 'value5', 'item6': 'value6'}]},
+    ])
+    def test_merge_dict_group_multiple(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_var'], {
+            'inventory_hostname': 'host1',
+            'hostvars': {
+                'host1': {
+                    'group_names': ['dummy1'],
+                    'inventory_hostname': 'host1',
+                    '1testlist__merge_var': {
+                        'var': [{'item1': 'value1', 'item2': 'value2'}]
+                    }
+                },
+                'host2': {
+                    'group_names': ['dummy2'],
+                    'inventory_hostname': 'host2',
+                    '2otherlist__merge_var': {
+                        'var': [{'item5': 'value5', 'item6': 'value6'}]
+                    }
+                },
+                'host3': {
+                    'group_names': ['dummy3'],
+                    'inventory_hostname': 'host3',
+                    '3otherlist__merge_var': {
+                        'var': [{'item3': 'value3', 'item4': 'value4'}]
+                    }
+                }
+            }
+        })
+
+        self.assertEqual(results, [
+            {'var': [
+                {'item1': 'value1', 'item2': 'value2'},
+                {'item5': 'value5', 'item6': 'value6'}
+            ]}
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1', 'dummy2']])
+    @patch.object(Templar, 'template', side_effect=[
+        ['item1'],
+        ['item5'],
+    ])
+    def test_merge_list_group_multiple(self, mock_set_options, mock_get_option, mock_template):
+        print()
+        results = self.merge_vars_lookup.run(['__merge_var'], {
+            'inventory_hostname': 'host1',
+            'hostvars': {
+                'host1': {
+                    'group_names': ['dummy1'],
+                    'inventory_hostname': 'host1',
+                    '1testlist__merge_var': ['item1']
+                },
+                'host2': {
+                    'group_names': ['dummy2'],
+                    'inventory_hostname': 'host2',
+                    '2otherlist__merge_var': ['item5']
+                },
+                'host3': {
+                    'group_names': ['dummy3'],
+                    'inventory_hostname': 'host3',
+                    '3otherlist__merge_var': ['item3']
+                }
+            }
+        })
+
+        self.assertEqual(results, [['item1', 'item5']])