mirror of
https://github.com/ansible-collections/community.general.git
synced 2024-09-14 20:13:21 +02:00
feat(lookup/merge_variables): Add all hosts mode to collect configuration across multiple hosts (#7999)
* Add Feature to collect variables accross different hosts * fix merging lists * adjust unit tests * lint fixes * adjusting integration tests * remove white spaces * Update plugins/lookup/merge_variables.py Co-authored-by: Felix Fontein <felix@fontein.de> * Update plugins/lookup/merge_variables.py Co-authored-by: Felix Fontein <felix@fontein.de> * Update plugins/lookup/merge_variables.py Co-authored-by: Felix Fontein <felix@fontein.de> * apply suggested changes to correctly handling the initial_value parameter, incl. additional test * whitespace --------- Co-authored-by: Alexander Petrenz <alexander.petrenz@posteo.de> Co-authored-by: Felix Fontein <felix@fontein.de>
This commit is contained in:
Alexander Petrenz
GitHub
parent
fa30b02294
commit
0ded1109fe
6 changed files with 302 additions and 12 deletions
2
.github/BOTMETA.yml
vendored
2
.github/BOTMETA.yml
vendored
|
|
@ -265,7 +265,7 @@ files:
|
|||
labels: manifold
|
||||
maintainers: galanoff
|
||||
$lookups/merge_variables.py:
|
||||
maintainers: rlenferink m-a-r-k-e
|
||||
maintainers: rlenferink m-a-r-k-e alpex8
|
||||
$lookups/onepass:
|
||||
labels: onepassword
|
||||
maintainers: samdoran
|
||||
|
|
|
|||
|
|
@ -10,11 +10,12 @@ DOCUMENTATION = """
|
|||
author:
|
||||
- Roy Lenferink (@rlenferink)
|
||||
- Mark Ettema (@m-a-r-k-e)
|
||||
- Alexander Petrenz (@alpex8)
|
||||
name: merge_variables
|
||||
short_description: merge variables with a certain suffix
|
||||
description:
|
||||
- This lookup returns the merged result of all variables in scope that match the given prefixes, suffixes, or
|
||||
regular expressions, optionally.
|
||||
regular expressions, optionally.
|
||||
version_added: 6.5.0
|
||||
options:
|
||||
_terms:
|
||||
|
|
@ -61,6 +62,13 @@ DOCUMENTATION = """
|
|||
ini:
|
||||
- section: merge_variables_lookup
|
||||
key: override
|
||||
groups:
|
||||
description:
|
||||
- Search for variables accross hosts that belong to the given groups. This allows to collect configuration pieces
|
||||
accross different hosts (for example a service on a host with its database on another host).
|
||||
type: list
|
||||
elements: str
|
||||
version_added: 8.5.0
|
||||
"""
|
||||
|
||||
EXAMPLES = """
|
||||
|
|
@ -131,22 +139,39 @@ def _verify_and_get_type(variable):
|
|||
|
||||
|
||||
class LookupModule(LookupBase):
|
||||
|
||||
def run(self, terms, variables=None, **kwargs):
|
||||
self.set_options(direct=kwargs)
|
||||
initial_value = self.get_option("initial_value", None)
|
||||
self._override = self.get_option('override', 'error')
|
||||
self._pattern_type = self.get_option('pattern_type', 'regex')
|
||||
self._groups = self.get_option('groups', None)
|
||||
|
||||
ret = []
|
||||
for term in terms:
|
||||
if not isinstance(term, str):
|
||||
raise AnsibleError("Non-string type '{0}' passed, only 'str' types are allowed!".format(type(term)))
|
||||
|
||||
ret.append(self._merge_vars(term, initial_value, variables))
|
||||
if not self._groups: # consider only own variables
|
||||
ret.append(self._merge_vars(term, initial_value, variables))
|
||||
else: # consider variables of hosts in given groups
|
||||
cross_host_merge_result = initial_value
|
||||
for host in variables["hostvars"]:
|
||||
if self._is_host_in_allowed_groups(variables["hostvars"][host]["group_names"]):
|
||||
cross_host_merge_result = self._merge_vars(term, cross_host_merge_result, variables["hostvars"][host])
|
||||
ret.append(cross_host_merge_result)
|
||||
|
||||
return ret
|
||||
|
||||
def _is_host_in_allowed_groups(self, host_groups):
|
||||
if 'all' in self._groups:
|
||||
return True
|
||||
|
||||
group_intersection = [host_group_name for host_group_name in host_groups if host_group_name in self._groups]
|
||||
if group_intersection:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _var_matches(self, key, search_pattern):
|
||||
if self._pattern_type == "prefix":
|
||||
return key.startswith(search_pattern)
|
||||
|
|
@ -162,7 +187,6 @@ class LookupModule(LookupBase):
|
|||
display.vvv("Merge variables with {0}: {1}".format(self._pattern_type, search_pattern))
|
||||
var_merge_names = sorted([key for key in variables.keys() if self._var_matches(key, search_pattern)])
|
||||
display.vvv("The following variables will be merged: {0}".format(var_merge_names))
|
||||
|
||||
prev_var_type = None
|
||||
result = None
|
||||
|
||||
|
|
|
|||
|
|
@ -11,3 +11,6 @@ ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
|
|||
ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
|
||||
ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \
|
||||
ansible-playbook test_with_env.yml "$@"
|
||||
|
||||
ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
|
||||
ansible-playbook -i test_inventory_all_hosts.yml test_all_hosts.yml "$@"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
---
|
||||
# Copyright (c) 2020, Thales Netherlands
|
||||
# Copyright (c) 2021, Ansible Project
|
||||
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
- name: Test merge_variables lookup plugin (multiple hosts)
|
||||
hosts: host0
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: Test merge dicts via all group
|
||||
delegate_to: localhost
|
||||
vars:
|
||||
merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_ex', pattern_type='suffix', groups=['all']) }}"
|
||||
block:
|
||||
- name: Test merge dicts via all group - Print the merged dict
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ merged_dict }}"
|
||||
|
||||
- name: Test merge dicts via all group - Validate that the dict is complete
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "(merged_dict.keys() | list | length) == 4"
|
||||
- "'item1' in merged_dict"
|
||||
- "'item2' in merged_dict"
|
||||
- "'item3' in merged_dict"
|
||||
- "'list_item' in merged_dict"
|
||||
- "merged_dict.list_item | length == 3"
|
||||
- name: Test merge dicts via two of three groups
|
||||
delegate_to: localhost
|
||||
vars:
|
||||
merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', pattern_type='suffix', groups=['dummy1', 'dummy2']) }}"
|
||||
block:
|
||||
- name: Test merge dicts via two of three groups - Print the merged dict
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ merged_dict }}"
|
||||
|
||||
- name: Test merge dicts via two of three groups - Validate that the dict is complete
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "(merged_dict.keys() | list | length) == 3"
|
||||
- "'item1' in merged_dict"
|
||||
- "'item2' in merged_dict"
|
||||
- "'list_item' in merged_dict"
|
||||
- "merged_dict.list_item | length == 2"
|
||||
- name: Test merge dicts via two of three groups with inital value
|
||||
delegate_to: localhost
|
||||
vars:
|
||||
initial_dict:
|
||||
initial: initial_value
|
||||
merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', initial_value=initial_dict, pattern_type='suffix', groups=['dummy1', 'dummy2']) }}"
|
||||
block:
|
||||
- name: Test merge dicts via two of three groups with inital value - Print the merged dict
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ merged_dict }}"
|
||||
|
||||
- name: Test merge dicts via two of three groups with inital value - Validate that the dict is complete
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "(merged_dict.keys() | list | length) == 4"
|
||||
- "'item1' in merged_dict"
|
||||
- "'item2' in merged_dict"
|
||||
- "'list_item' in merged_dict"
|
||||
- "merged_dict.list_item | length == 2"
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
# Copyright (c) 2020, Thales Netherlands
|
||||
# Copyright (c) 2021, Ansible Project
|
||||
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
all:
|
||||
hosts:
|
||||
host0:
|
||||
host1:
|
||||
testdict1__merge_dict_ex:
|
||||
item1: value1
|
||||
list_item:
|
||||
- test1
|
||||
|
||||
testdict2__merge_dict_ex:
|
||||
item2: value2
|
||||
list_item:
|
||||
- test2
|
||||
|
||||
testdict__merge_dict_in:
|
||||
item1: value1
|
||||
list_item:
|
||||
- test1
|
||||
host2:
|
||||
testdict3__merge_dict_ex:
|
||||
item3: value3
|
||||
list_item:
|
||||
- test3
|
||||
|
||||
testdict__merge_dict_in:
|
||||
item2: value2
|
||||
list_item:
|
||||
- test2
|
||||
|
||||
host3:
|
||||
testdict__merge_dict_in:
|
||||
item3: value3
|
||||
list_item:
|
||||
- test3
|
||||
|
||||
dummy1:
|
||||
hosts:
|
||||
host1:
|
||||
|
||||
dummy2:
|
||||
hosts:
|
||||
host2:
|
||||
|
||||
dummy3:
|
||||
hosts:
|
||||
host3:
|
||||
|
|
@ -24,7 +24,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar)
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
|
||||
def test_merge_list(self, mock_set_options, mock_get_option, mock_template):
|
||||
results = self.merge_vars_lookup.run(['__merge_list'], {
|
||||
|
|
@ -36,7 +36,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
self.assertEqual(results, [['item1', 'item3']])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix'])
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
|
||||
def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
|
||||
results = self.merge_vars_lookup.run(['__merge_list'], {
|
||||
|
|
@ -48,7 +48,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
self.assertEqual(results, [['initial_item', 'item1', 'item3']])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
|
||||
{'item2': 'test', 'list_item': ['test2']}])
|
||||
def test_merge_dict(self, mock_set_options, mock_get_option, mock_template):
|
||||
|
|
@ -73,7 +73,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']},
|
||||
'ignore', 'suffix'])
|
||||
'ignore', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
|
||||
{'item2': 'test', 'list_item': ['test2']}])
|
||||
def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
|
||||
|
|
@ -98,7 +98,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix'])
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
|
||||
@patch.object(Display, 'warning')
|
||||
def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display):
|
||||
|
|
@ -111,7 +111,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
self.assertEqual(results, [{'item': 'value2'}])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix'])
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
|
||||
def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template):
|
||||
with self.assertRaises(AnsibleError):
|
||||
|
|
@ -121,7 +121,7 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
})
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', None])
|
||||
@patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
|
||||
['item2', 'item3']])
|
||||
def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template):
|
||||
|
|
@ -133,3 +133,150 @@ class TestMergeVariablesLookup(unittest.TestCase):
|
|||
},
|
||||
'testdict__merge_var': ['item2', 'item3']
|
||||
})
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['all']])
|
||||
@patch.object(Templar, 'template', side_effect=[
|
||||
{'var': [{'item1': 'value1', 'item2': 'value2'}]},
|
||||
{'var': [{'item5': 'value5', 'item6': 'value6'}]},
|
||||
])
|
||||
def test_merge_dict_group_all(self, mock_set_options, mock_get_option, mock_template):
|
||||
results = self.merge_vars_lookup.run(['__merge_var'], {
|
||||
'inventory_hostname': 'host1',
|
||||
'hostvars': {
|
||||
'host1': {
|
||||
'group_names': ['dummy1'],
|
||||
'inventory_hostname': 'host1',
|
||||
'1testlist__merge_var': {
|
||||
'var': [{'item1': 'value1', 'item2': 'value2'}]
|
||||
}
|
||||
},
|
||||
'host2': {
|
||||
'group_names': ['dummy1'],
|
||||
'inventory_hostname': 'host2',
|
||||
'2otherlist__merge_var': {
|
||||
'var': [{'item5': 'value5', 'item6': 'value6'}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
self.assertEqual(results, [
|
||||
{'var': [
|
||||
{'item1': 'value1', 'item2': 'value2'},
|
||||
{'item5': 'value5', 'item6': 'value6'}
|
||||
]}
|
||||
])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1']])
|
||||
@patch.object(Templar, 'template', side_effect=[
|
||||
{'var': [{'item1': 'value1', 'item2': 'value2'}]},
|
||||
{'var': [{'item5': 'value5', 'item6': 'value6'}]},
|
||||
])
|
||||
def test_merge_dict_group_single(self, mock_set_options, mock_get_option, mock_template):
|
||||
results = self.merge_vars_lookup.run(['__merge_var'], {
|
||||
'inventory_hostname': 'host1',
|
||||
'hostvars': {
|
||||
'host1': {
|
||||
'group_names': ['dummy1'],
|
||||
'inventory_hostname': 'host1',
|
||||
'1testlist__merge_var': {
|
||||
'var': [{'item1': 'value1', 'item2': 'value2'}]
|
||||
}
|
||||
},
|
||||
'host2': {
|
||||
'group_names': ['dummy1'],
|
||||
'inventory_hostname': 'host2',
|
||||
'2otherlist__merge_var': {
|
||||
'var': [{'item5': 'value5', 'item6': 'value6'}]
|
||||
}
|
||||
},
|
||||
'host3': {
|
||||
'group_names': ['dummy2'],
|
||||
'inventory_hostname': 'host3',
|
||||
'3otherlist__merge_var': {
|
||||
'var': [{'item3': 'value3', 'item4': 'value4'}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
self.assertEqual(results, [
|
||||
{'var': [
|
||||
{'item1': 'value1', 'item2': 'value2'},
|
||||
{'item5': 'value5', 'item6': 'value6'}
|
||||
]}
|
||||
])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1', 'dummy2']])
|
||||
@patch.object(Templar, 'template', side_effect=[
|
||||
{'var': [{'item1': 'value1', 'item2': 'value2'}]},
|
||||
{'var': [{'item5': 'value5', 'item6': 'value6'}]},
|
||||
])
|
||||
def test_merge_dict_group_multiple(self, mock_set_options, mock_get_option, mock_template):
|
||||
results = self.merge_vars_lookup.run(['__merge_var'], {
|
||||
'inventory_hostname': 'host1',
|
||||
'hostvars': {
|
||||
'host1': {
|
||||
'group_names': ['dummy1'],
|
||||
'inventory_hostname': 'host1',
|
||||
'1testlist__merge_var': {
|
||||
'var': [{'item1': 'value1', 'item2': 'value2'}]
|
||||
}
|
||||
},
|
||||
'host2': {
|
||||
'group_names': ['dummy2'],
|
||||
'inventory_hostname': 'host2',
|
||||
'2otherlist__merge_var': {
|
||||
'var': [{'item5': 'value5', 'item6': 'value6'}]
|
||||
}
|
||||
},
|
||||
'host3': {
|
||||
'group_names': ['dummy3'],
|
||||
'inventory_hostname': 'host3',
|
||||
'3otherlist__merge_var': {
|
||||
'var': [{'item3': 'value3', 'item4': 'value4'}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
self.assertEqual(results, [
|
||||
{'var': [
|
||||
{'item1': 'value1', 'item2': 'value2'},
|
||||
{'item5': 'value5', 'item6': 'value6'}
|
||||
]}
|
||||
])
|
||||
|
||||
@patch.object(AnsiblePlugin, 'set_options')
|
||||
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix', ['dummy1', 'dummy2']])
|
||||
@patch.object(Templar, 'template', side_effect=[
|
||||
['item1'],
|
||||
['item5'],
|
||||
])
|
||||
def test_merge_list_group_multiple(self, mock_set_options, mock_get_option, mock_template):
|
||||
print()
|
||||
results = self.merge_vars_lookup.run(['__merge_var'], {
|
||||
'inventory_hostname': 'host1',
|
||||
'hostvars': {
|
||||
'host1': {
|
||||
'group_names': ['dummy1'],
|
||||
'inventory_hostname': 'host1',
|
||||
'1testlist__merge_var': ['item1']
|
||||
},
|
||||
'host2': {
|
||||
'group_names': ['dummy2'],
|
||||
'inventory_hostname': 'host2',
|
||||
'2otherlist__merge_var': ['item5']
|
||||
},
|
||||
'host3': {
|
||||
'group_names': ['dummy3'],
|
||||
'inventory_hostname': 'host3',
|
||||
'3otherlist__merge_var': ['item3']
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
self.assertEqual(results, [['item1', 'item5']])
|
||||
|
|
|
|||
Loading…
Reference in a new issue