From 3e235f78d12cd6ab680f95159f45dd4881fcae8b Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 12:42:22 +0000 Subject: [PATCH] [PR #7999/0ded1109 backport][stable-8] feat(lookup/merge_variables): Add all hosts mode to collect configuration across multiple hosts (#8069) feat(lookup/merge_variables): Add all hosts mode to collect configuration across multiple hosts (#7999) * Add Feature to collect variables accross different hosts * fix merging lists * adjust unit tests * lint fixes * adjusting integration tests * remove white spaces * Update plugins/lookup/merge_variables.py Co-authored-by: Felix Fontein * Update plugins/lookup/merge_variables.py Co-authored-by: Felix Fontein * Update plugins/lookup/merge_variables.py Co-authored-by: Felix Fontein * apply suggested changes to correctly handling the initial_value parameter, incl. additional test * whitespace --------- Co-authored-by: Alexander Petrenz Co-authored-by: Felix Fontein (cherry picked from commit 0ded1109fecee3a871be52ea02c48406d803765c) Co-authored-by: Alexander Petrenz --- .github/BOTMETA.yml | 2 +- plugins/lookup/merge_variables.py | 32 +++- .../targets/lookup_merge_variables/runme.sh | 3 + .../lookup_merge_variables/test_all_hosts.yml | 64 +++++++ .../test_inventory_all_hosts.yml | 52 ++++++ .../plugins/lookup/test_merge_variables.py | 161 +++++++++++++++++- 6 files changed, 302 insertions(+), 12 deletions(-) create mode 100644 tests/integration/targets/lookup_merge_variables/test_all_hosts.yml create mode 100644 tests/integration/targets/lookup_merge_variables/test_inventory_all_hosts.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index a9e871d192..281e55435c 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -265,7 +265,7 @@ files: labels: manifold maintainers: galanoff $lookups/merge_variables.py: - maintainers: rlenferink m-a-r-k-e + maintainers: rlenferink m-a-r-k-e alpex8 $lookups/onepass: labels: onepassword maintainers: samdoran diff --git a/plugins/lookup/merge_variables.py b/plugins/lookup/merge_variables.py index dbd9401b10..4fc33014c0 100644 --- a/plugins/lookup/merge_variables.py +++ b/plugins/lookup/merge_variables.py @@ -10,11 +10,12 @@ DOCUMENTATION = """ author: - Roy Lenferink (@rlenferink) - Mark Ettema (@m-a-r-k-e) + - Alexander Petrenz (@alpex8) name: merge_variables short_description: merge variables with a certain suffix description: - This lookup returns the merged result of all variables in scope that match the given prefixes, suffixes, or - regular expressions, optionally. + regular expressions, optionally. version_added: 6.5.0 options: _terms: @@ -61,6 +62,13 @@ DOCUMENTATION = """ ini: - section: merge_variables_lookup key: override + groups: + description: + - Search for variables accross hosts that belong to the given groups. This allows to collect configuration pieces + accross different hosts (for example a service on a host with its database on another host). + type: list + elements: str + version_added: 8.5.0 """ EXAMPLES = """ @@ -131,22 +139,39 @@ def _verify_and_get_type(variable): class LookupModule(LookupBase): - def run(self, terms, variables=None, **kwargs): self.set_options(direct=kwargs) initial_value = self.get_option("initial_value", None) self._override = self.get_option('override', 'error') self._pattern_type = self.get_option('pattern_type', 'regex') + self._groups = self.get_option('groups', None) ret = [] for term in terms: if not isinstance(term, str): raise AnsibleError("Non-string type '{0}' passed, only 'str' types are allowed!".format(type(term))) - ret.append(self._merge_vars(term, initial_value, variables)) + if not self._groups: # consider only own variables + ret.append(self._merge_vars(term, initial_value, variables)) + else: # consider variables of hosts in given groups + cross_host_merge_result = initial_value + for host in variables["hostvars"]: + if self._is_host_in_allowed_groups(variables["hostvars"][host]["group_names"]): + cross_host_merge_result = self._merge_vars(term, cross_host_merge_result, variables["hostvars"][host]) + ret.append(cross_host_merge_result) return ret + def _is_host_in_allowed_groups(self, host_groups): + if 'all' in self._groups: + return True + + group_intersection = [host_group_name for host_group_name in host_groups if host_group_name in self._groups] + if group_intersection: + return True + + return False + def _var_matches(self, key, search_pattern): if self._pattern_type == "prefix": return key.startswith(search_pattern) @@ -162,7 +187,6 @@ class LookupModule(LookupBase): display.vvv("Merge variables with {0}: {1}".format(self._pattern_type, search_pattern)) var_merge_names = sorted([key for key in variables.keys() if self._var_matches(key, search_pattern)]) display.vvv("The following variables will be merged: {0}".format(var_merge_names)) - prev_var_type = None result = None diff --git a/tests/integration/targets/lookup_merge_variables/runme.sh b/tests/integration/targets/lookup_merge_variables/runme.sh index 52a38f4a54..4e66476be4 100755 --- a/tests/integration/targets/lookup_merge_variables/runme.sh +++ b/tests/integration/targets/lookup_merge_variables/runme.sh @@ -11,3 +11,6 @@ ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \ ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \ ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \ ansible-playbook test_with_env.yml "$@" + +ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \ + ansible-playbook -i test_inventory_all_hosts.yml test_all_hosts.yml "$@" diff --git a/tests/integration/targets/lookup_merge_variables/test_all_hosts.yml b/tests/integration/targets/lookup_merge_variables/test_all_hosts.yml new file mode 100644 index 0000000000..3070087bb9 --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/test_all_hosts.yml @@ -0,0 +1,64 @@ +--- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test merge_variables lookup plugin (multiple hosts) + hosts: host0 + gather_facts: false + tasks: + - name: Test merge dicts via all group + delegate_to: localhost + vars: + merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_ex', pattern_type='suffix', groups=['all']) }}" + block: + - name: Test merge dicts via all group - Print the merged dict + ansible.builtin.debug: + msg: "{{ merged_dict }}" + + - name: Test merge dicts via all group - Validate that the dict is complete + ansible.builtin.assert: + that: + - "(merged_dict.keys() | list | length) == 4" + - "'item1' in merged_dict" + - "'item2' in merged_dict" + - "'item3' in merged_dict" + - "'list_item' in merged_dict" + - "merged_dict.list_item | length == 3" + - name: Test merge dicts via two of three groups + delegate_to: localhost + vars: + merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', pattern_type='suffix', groups=['dummy1', 'dummy2']) }}" + block: + - name: Test merge dicts via two of three groups - Print the merged dict + ansible.builtin.debug: + msg: "{{ merged_dict }}" + + - name: Test merge dicts via two of three groups - Validate that the dict is complete + ansible.builtin.assert: + that: + - "(merged_dict.keys() | list | length) == 3" + - "'item1' in merged_dict" + - "'item2' in merged_dict" + - "'list_item' in merged_dict" + - "merged_dict.list_item | length == 2" + - name: Test merge dicts via two of three groups with inital value + delegate_to: localhost + vars: + initial_dict: + initial: initial_value + merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', initial_value=initial_dict, pattern_type='suffix', groups=['dummy1', 'dummy2']) }}" + block: + - name: Test merge dicts via two of three groups with inital value - Print the merged dict + ansible.builtin.debug: + msg: "{{ merged_dict }}" + + - name: Test merge dicts via two of three groups with inital value - Validate that the dict is complete + ansible.builtin.assert: + that: + - "(merged_dict.keys() | list | length) == 4" + - "'item1' in merged_dict" + - "'item2' in merged_dict" + - "'list_item' in merged_dict" + - "merged_dict.list_item | length == 2" diff --git a/tests/integration/targets/lookup_merge_variables/test_inventory_all_hosts.yml b/tests/integration/targets/lookup_merge_variables/test_inventory_all_hosts.yml new file mode 100644 index 0000000000..edf5a9e46d --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/test_inventory_all_hosts.yml @@ -0,0 +1,52 @@ +--- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +all: + hosts: + host0: + host1: + testdict1__merge_dict_ex: + item1: value1 + list_item: + - test1 + + testdict2__merge_dict_ex: + item2: value2 + list_item: + - test2 + + testdict__merge_dict_in: + item1: value1 + list_item: + - test1 + host2: + testdict3__merge_dict_ex: + item3: value3 + list_item: + - test3 + + testdict__merge_dict_in: + item2: value2 + list_item: + - test2 + + host3: + testdict__merge_dict_in: + item3: value3 + list_item: + - test3 + +dummy1: + hosts: + host1: + +dummy2: + hosts: + host2: + +dummy3: + hosts: + host3: diff --git a/tests/unit/plugins/lookup/test_merge_variables.py b/tests/unit/plugins/lookup/test_merge_variables.py index 5085797b3e..66cb2f08bb 100644 --- a/tests/unit/plugins/lookup/test_merge_variables.py +++ b/tests/unit/plugins/lookup/test_merge_variables.py @@ -24,7 +24,7 @@ class TestMergeVariablesLookup(unittest.TestCase): self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar) @patch.object(AnsiblePlugin, 'set_options') - @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix']) + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']]) def test_merge_list(self, mock_set_options, mock_get_option, mock_template): results = self.merge_vars_lookup.run(['__merge_list'], { @@ -36,7 +36,7 @@ class TestMergeVariablesLookup(unittest.TestCase): self.assertEqual(results, [['item1', 'item3']]) @patch.object(AnsiblePlugin, 'set_options') - @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix']) + @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']]) def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template): results = self.merge_vars_lookup.run(['__merge_list'], { @@ -48,7 +48,7 @@ class TestMergeVariablesLookup(unittest.TestCase): self.assertEqual(results, [['initial_item', 'item1', 'item3']]) @patch.object(AnsiblePlugin, 'set_options') - @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix']) + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']}, {'item2': 'test', 'list_item': ['test2']}]) def test_merge_dict(self, mock_set_options, mock_get_option, mock_template): @@ -73,7 +73,7 @@ class TestMergeVariablesLookup(unittest.TestCase): @patch.object(AnsiblePlugin, 'set_options') @patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']}, - 'ignore', 'suffix']) + 'ignore', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']}, {'item2': 'test', 'list_item': ['test2']}]) def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template): @@ -98,7 +98,7 @@ class TestMergeVariablesLookup(unittest.TestCase): ]) @patch.object(AnsiblePlugin, 'set_options') - @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix']) + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}]) @patch.object(Display, 'warning') def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display): @@ -111,7 +111,7 @@ class TestMergeVariablesLookup(unittest.TestCase): self.assertEqual(results, [{'item': 'value2'}]) @patch.object(AnsiblePlugin, 'set_options') - @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix']) + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}]) def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template): with self.assertRaises(AnsibleError): @@ -121,7 +121,7 @@ class TestMergeVariablesLookup(unittest.TestCase): }) @patch.object(AnsiblePlugin, 'set_options') - @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix']) + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None]) @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']}, ['item2', 'item3']]) def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template): @@ -133,3 +133,150 @@ class TestMergeVariablesLookup(unittest.TestCase): }, 'testdict__merge_var': ['item2', 'item3'] }) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['all']]) + @patch.object(Templar, 'template', side_effect=[ + {'var': [{'item1': 'value1', 'item2': 'value2'}]}, + {'var': [{'item5': 'value5', 'item6': 'value6'}]}, + ]) + def test_merge_dict_group_all(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_var'], { + 'inventory_hostname': 'host1', + 'hostvars': { + 'host1': { + 'group_names': ['dummy1'], + 'inventory_hostname': 'host1', + '1testlist__merge_var': { + 'var': [{'item1': 'value1', 'item2': 'value2'}] + } + }, + 'host2': { + 'group_names': ['dummy1'], + 'inventory_hostname': 'host2', + '2otherlist__merge_var': { + 'var': [{'item5': 'value5', 'item6': 'value6'}] + } + } + } + }) + + self.assertEqual(results, [ + {'var': [ + {'item1': 'value1', 'item2': 'value2'}, + {'item5': 'value5', 'item6': 'value6'} + ]} + ]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1']]) + @patch.object(Templar, 'template', side_effect=[ + {'var': [{'item1': 'value1', 'item2': 'value2'}]}, + {'var': [{'item5': 'value5', 'item6': 'value6'}]}, + ]) + def test_merge_dict_group_single(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_var'], { + 'inventory_hostname': 'host1', + 'hostvars': { + 'host1': { + 'group_names': ['dummy1'], + 'inventory_hostname': 'host1', + '1testlist__merge_var': { + 'var': [{'item1': 'value1', 'item2': 'value2'}] + } + }, + 'host2': { + 'group_names': ['dummy1'], + 'inventory_hostname': 'host2', + '2otherlist__merge_var': { + 'var': [{'item5': 'value5', 'item6': 'value6'}] + } + }, + 'host3': { + 'group_names': ['dummy2'], + 'inventory_hostname': 'host3', + '3otherlist__merge_var': { + 'var': [{'item3': 'value3', 'item4': 'value4'}] + } + } + } + }) + + self.assertEqual(results, [ + {'var': [ + {'item1': 'value1', 'item2': 'value2'}, + {'item5': 'value5', 'item6': 'value6'} + ]} + ]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1', 'dummy2']]) + @patch.object(Templar, 'template', side_effect=[ + {'var': [{'item1': 'value1', 'item2': 'value2'}]}, + {'var': [{'item5': 'value5', 'item6': 'value6'}]}, + ]) + def test_merge_dict_group_multiple(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_var'], { + 'inventory_hostname': 'host1', + 'hostvars': { + 'host1': { + 'group_names': ['dummy1'], + 'inventory_hostname': 'host1', + '1testlist__merge_var': { + 'var': [{'item1': 'value1', 'item2': 'value2'}] + } + }, + 'host2': { + 'group_names': ['dummy2'], + 'inventory_hostname': 'host2', + '2otherlist__merge_var': { + 'var': [{'item5': 'value5', 'item6': 'value6'}] + } + }, + 'host3': { + 'group_names': ['dummy3'], + 'inventory_hostname': 'host3', + '3otherlist__merge_var': { + 'var': [{'item3': 'value3', 'item4': 'value4'}] + } + } + } + }) + + self.assertEqual(results, [ + {'var': [ + {'item1': 'value1', 'item2': 'value2'}, + {'item5': 'value5', 'item6': 'value6'} + ]} + ]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1', 'dummy2']]) + @patch.object(Templar, 'template', side_effect=[ + ['item1'], + ['item5'], + ]) + def test_merge_list_group_multiple(self, mock_set_options, mock_get_option, mock_template): + print() + results = self.merge_vars_lookup.run(['__merge_var'], { + 'inventory_hostname': 'host1', + 'hostvars': { + 'host1': { + 'group_names': ['dummy1'], + 'inventory_hostname': 'host1', + '1testlist__merge_var': ['item1'] + }, + 'host2': { + 'group_names': ['dummy2'], + 'inventory_hostname': 'host2', + '2otherlist__merge_var': ['item5'] + }, + 'host3': { + 'group_names': ['dummy3'], + 'inventory_hostname': 'host3', + '3otherlist__merge_var': ['item3'] + } + } + }) + + self.assertEqual(results, [['item1', 'item5']])