ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

Add new merge_variables lookup plugin (#5533)

Browse source 
* Add new merge_variables lookup plugin

* Add changelog fragment

* Process bot feedback

* Refactor override options and add pattern_type option

* Fix unit tests

* Changed default pattern_type and simplified plugin based on feedback

* Processed feedback for merge_variables lookup plugin

* Adjust version_added.

---------

Co-authored-by: Felix Fontein <felix@fontein.de>
This commit is contained in:
Roy Lenferink 2023-03-26 11:27:30 +02:00 committed by GitHub
parent be7f11bf39
commit f52dd194f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 620 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/BOTMETA.yml vendored
View file

@ -241,6 +241,8 @@ files:
$lookups/manifold.py:
labels: manifold
maintainers: galanoff
$lookups/merge_variables.py:
maintainers: rlenferink m-a-r-k-e
$lookups/onepass:
labels: onepassword
maintainers: samdoran

212
plugins/lookup/merge_variables.py Normal file
View file

@ -0,0 +1,212 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Thales Netherlands
# Copyright (c) 2021, Ansible Project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
DOCUMENTATION = """
author:
- Roy Lenferink (@rlenferink)
- Mark Ettema (@m-a-r-k-e)
name: merge_variables
short_description: merge variables with a certain suffix
description:
- This lookup returns the merged result of all variables in scope that match the given prefixes, suffixes, or
regular expressions, optionally.
version_added: 6.5.0
options:
_terms:
description:
- Depending on the value of I(pattern_type), this is a list of prefixes, suffixes, or regular expressions
that will be used to match all variables that should be merged.
required: true
type: list
elements: str
pattern_type:
description:
- Change the way of searching for the specified pattern.
type: str
default: 'regex'
choices:
- prefix
- suffix
- regex
env:
- name: ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE
ini:
- section: merge_variables_lookup
key: pattern_type
initial_value:
description:
- An initial value to start with.
type: raw
override:
description:
- Return an error, print a warning or ignore it when a key will be overwritten.
- The default behavior C(error) makes the plugin fail when a key would be overwritten.
- When C(warn) and C(ignore) are used, note that it is important to know that the variables
are sorted by name before being merged. Keys for later variables in this order will overwrite
keys of the same name for variables earlier in this order. To avoid potential confusion,
better use I(override=error) whenever possible.
type: str
default: 'error'
choices:
- error
- warn
- ignore
env:
- name: ANSIBLE_MERGE_VARIABLES_OVERRIDE
ini:
- section: merge_variables_lookup
key: override
"""
EXAMPLES = """
# Some example variables, they can be defined anywhere as long as they are in scope
test_init_list:
- "list init item 1"
- "list init item 2"
testa__test_list:
- "test a item 1"
testb__test_list:
- "test b item 1"
testa__test_dict:
ports:
- 1
testb__test_dict:
ports:
- 3
# Merge variables that end with '__test_dict' and store the result in a variable 'example_a'
example_a: "{{ lookup('community.general.merge_variables', '__test_dict') }}"
# The variable example_a now contains:
# ports:
# - 1
# - 3
# Merge variables that end with '__test_list', starting with an initial value and store the result
# in a variable 'example_b'
example_b: "{{ lookup('community.general.merge_variables', '__test_list', initial_value=test_init_list) }}"
# The variable example_b now contains:
# - "list init item 1"
# - "list init item 2"
# - "test a item 1"
# - "test b item 1"
"""
RETURN = """
_raw:
description: In case the search matches list items, a list will be returned. In case the search matches dicts, a
dict will be returned.
type: raw
elements: raw
"""
import re
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.display import Display
display = Display()
def _verify_and_get_type(variable):
if isinstance(variable, list):
return "list"
elif isinstance(variable, dict):
return "dict"
else:
raise AnsibleError("Not supported type detected, variable must be a list or a dict")
class LookupModule(LookupBase):
def run(self, terms, variables=None, **kwargs):
self.set_options(direct=kwargs)
initial_value = self.get_option("initial_value", None)
self._override = self.get_option('override', 'error')
self._pattern_type = self.get_option('pattern_type', 'regex')
ret = []
for term in terms:
if not isinstance(term, str):
raise AnsibleError("Non-string type '{0}' passed, only 'str' types are allowed!".format(type(term)))
ret.append(self._merge_vars(term, initial_value, variables))
return ret
def _var_matches(self, key, search_pattern):
if self._pattern_type == "prefix":
return key.startswith(search_pattern)
elif self._pattern_type == "suffix":
return key.endswith(search_pattern)
elif self._pattern_type == "regex":
matcher = re.compile(search_pattern)
return matcher.search(key)
return False
def _merge_vars(self, search_pattern, initial_value, variables):
display.vvv("Merge variables with {0}: {1}".format(self._pattern_type, search_pattern))
var_merge_names = sorted([key for key in variables.keys() if self._var_matches(key, search_pattern)])
display.vvv("The following variables will be merged: {0}".format(var_merge_names))
prev_var_type = None
result = None
if initial_value is not None:
prev_var_type = _verify_and_get_type(initial_value)
result = initial_value
for var_name in var_merge_names:
var_value = self._templar.template(variables[var_name]) # Render jinja2 templates
var_type = _verify_and_get_type(var_value)
if prev_var_type is None:
prev_var_type = var_type
elif prev_var_type != var_type:
raise AnsibleError("Unable to merge, not all variables are of the same type")
if result is None:
result = var_value
continue
if var_type == "dict":
result = self._merge_dict(var_value, result, [var_name])
else: # var_type == "list"
result += var_value
return result
def _merge_dict(self, src, dest, path):
for key, value in src.items():
if isinstance(value, dict):
node = dest.setdefault(key, {})
self._merge_dict(value, node, path + [key])
elif isinstance(value, list) and key in dest:
dest[key] += value
else:
if (key in dest) and dest[key] != value:
msg = "The key '{0}' with value '{1}' will be overwritten with value '{2}' from '{3}.{0}'".format(
key, dest[key], value, ".".join(path))
if self._override == "error":
raise AnsibleError(msg)
if self._override == "warn":
display.warning(msg)
dest[key] = value
return dest

6
tests/integration/targets/lookup_merge_variables/aliases Normal file
View file

@ -0,0 +1,6 @@
# Copyright (c) Ansible Project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
azp/posix/1
skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller

13
tests/integration/targets/lookup_merge_variables/runme.sh Executable file
View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Copyright (c) 2020, Thales Netherlands
# Copyright (c) 2021, Ansible project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
set -eux
ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
ansible-playbook test.yml "$@"
ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \
ansible-playbook test_with_env.yml "$@"

174
tests/integration/targets/lookup_merge_variables/test.yml Normal file
View file

@ -0,0 +1,174 @@
---
# Copyright (c) 2020, Thales Netherlands
# Copyright (c) 2021, Ansible Project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
- name: Test merge_variables lookup plugin
hosts: localhost
tasks:
- name: Include test data
include_vars: vars.yml
# Test the default behavior
- name: Test merge list
block:
- name: Print the merged list
debug:
msg: "{{ merged_list }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list | length) == 2"
- "'item1' in merged_list"
- "'item3' in merged_list"
vars:
merged_list: "{{ lookup('community.general.merge_variables', '^.+__merge_list$') }}"
- name: Test merge dict
block:
- name: Print the merged list
debug:
msg: "{{ merged_dict }}"
- name: Validate that dict is complete
assert:
that:
- "'item1' in merged_dict"
- "'item2' in merged_dict"
- "'list_item' in merged_dict"
- "(merged_dict.list_item | length) == 2"
- "'test1' in (merged_dict.list_item)"
- "'test2' in (merged_dict.list_item)"
vars:
merged_dict: "{{ lookup('community.general.merge_variables', '^.+__merge_dict$') }}"
# Test the behavior when no results are found
- name: Test merge without results
block:
- debug:
msg: "{{ not_found }}"
- name: Validate that the variable defaults to an empty list
vars:
assert:
that:
- "(not_found | default('default-used', True)) == 'default-used'"
vars:
not_found: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$') }}"
# Test the 'pattern_type' options
- name: Test merge list (pattern_type = prefix)
block:
- name: Print the merged list
debug:
msg: "{{ merged_list }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list | length) == 4"
- "'item1' in merged_list"
- "'item2' in merged_list"
- "'item2' in merged_list"
- "'item3' in merged_list"
vars:
merged_list: "{{ lookup('community.general.merge_variables', 'testlist', pattern_type='prefix') }}"
- name: Test merge list (pattern_type = suffix)
block:
- name: Print the merged list
debug:
msg: "{{ merged_list }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list | length) == 2"
- "'item1' in merged_list"
- "'item3' in merged_list"
vars:
merged_list: "{{ lookup('community.general.merge_variables', '__merge_list', pattern_type='suffix') }}"
- name: Test merge list (pattern_type = regex)
block:
- name: Print the merged list
debug:
msg: "{{ merged_list }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list | length) == 3"
- "'item1' in merged_list"
- "'item2' in merged_list"
- "'item3' in merged_list"
vars:
merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}"
# Test the 'initial_value' option
- name: Test merge without results but with initial value
block:
- name: Print the merged list
debug:
msg: "{{ not_found_initial_value }}"
- name: Validate that the variable only contains the initial value
vars:
assert:
that:
- "(not_found_initial_value | count) == 1"
- "(not_found_initial_value | first) == 'item2'"
vars:
not_found_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$', initial_value=testlist_initial_value) }}"
- name: Test merging a list with an initial value
block:
- name: Print the merged list
debug:
msg: "{{ merged_list_with_initial_value }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list_with_initial_value | length) == 3"
- "'item1' in merged_list_with_initial_value"
- "'item2' in merged_list_with_initial_value"
- "'item3' in merged_list_with_initial_value"
vars:
merged_list_with_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_list$', initial_value=testlist_initial_value) }}"
# Test the 'override' options
- name: Test the 'override=warn' option
block:
- name: Print the merged list
debug:
msg: "{{ merged_with_override_warn }}"
- name: Validate that the dict is complete and the warning is printed
assert:
that:
- "'key_to_override' in merged_with_override_warn"
- "merged_with_override_warn.key_to_override == 'Override value'"
- "'key_to_override' in lookup('file', logging_output_file)" # Check if a message is given
- "'[WARNING]' in lookup('file', logging_output_file)" # and verify that the message is a WARNING
vars:
merged_with_override_warn: "{{ lookup('community.general.merge_variables', '^.+__override_warn$', initial_value=override_warn_init, override='warn') }}"
- name: Test the 'override=error' option
block:
- name: Validate that an override result in an error
debug:
msg: "{{ lookup('community.general.merge_variables', '^.+__override_error$', initial_value=override_error_init, override='error') }}"
ignore_errors: true # Do not stop the playbook
register: _override_error_result
- name: Print the output
debug:
msg: "{{ _override_error_result }}"
- name: Validate that the error is reported
assert:
that:
- "_override_error_result.failed"
- "'key_to_override' in _override_error_result.msg"

44
tests/integration/targets/lookup_merge_variables/test_with_env.yml Normal file
View file

@ -0,0 +1,44 @@
---
# Copyright (c) 2020, Thales Netherlands
# Copyright (c) 2021, Ansible Project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
- name: Test merge_variables lookup plugin
hosts: localhost
tasks:
- name: Include test data
include_vars: vars.yml
# Test the pattern option using the environment variable
- name: Test merge list (pattern_type = regex)
block:
- name: Print the merged list
debug:
msg: "{{ merged_list }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list | length) == 2"
- "'item1' in merged_list"
- "'item3' in merged_list"
vars:
merged_list: "{{ lookup('community.general.merge_variables', '__merge_list') }}"
# Test whether the pattern option can be overridden
- name: Test merge list (pattern_type = suffix)
block:
- name: Print the merged list
debug:
msg: "{{ merged_list }}"
- name: Validate that the list is complete
assert:
that:
- "(merged_list | length) == 3"
- "'item1' in merged_list"
- "'item2' in merged_list"
- "'item3' in merged_list"
vars:
merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}"

34
tests/integration/targets/lookup_merge_variables/vars.yml Normal file
View file

@ -0,0 +1,34 @@
---
# Copyright (c) 2020, Thales Netherlands
# Copyright (c) 2021, Ansible Project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
testlist_initial_value: "{{ testlist2 }}"
testlist1__merge_list:
- item1
testlist2:
- item2
testlist3__merge_list:
- item3
testdict1__merge_dict:
item1: test
list_item:
- test1
testdict2__merge_dict:
item2: test
list_item:
- test2
override_warn_init:
key_to_override: Initial value
override__override_warn:
key_to_override: Override value
override_error_init:
key_to_override: Initial value
override__override_error:
key_to_override: Override value
logging_output_file: /tmp/ansible-test-merge-variables # The Ansible log output is available in this file

135
tests/unit/plugins/lookup/test_merge_variables.py Normal file
View file

@ -0,0 +1,135 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Thales Netherlands
# Copyright (c) 2021, Ansible Project
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
# SPDX-License-Identifier: GPL-3.0-or-later
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible_collections.community.general.tests.unit.compat import unittest
from ansible_collections.community.general.tests.unit.compat.mock import patch
from ansible_collections.community.general.tests.unit.mock.loader import DictDataLoader
from ansible.plugins import AnsiblePlugin
from ansible.template import Templar
from ansible.errors import AnsibleError
from ansible.utils.display import Display
from ansible_collections.community.general.plugins.lookup import merge_variables
class TestMergeVariablesLookup(unittest.TestCase):
def setUp(self):
self.loader = DictDataLoader({})
self.templar = Templar(loader=self.loader, variables={})
self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar)
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
@patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
def test_merge_list(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(['__merge_list'], {
'testlist1__merge_list': ['item1'],
'testlist2': ['item2'],
'testlist3__merge_list': ['item3']
})
self.assertEqual(results, [['item1', 'item3']])
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix'])
@patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(['__merge_list'], {
'testlist1__merge_list': ['item1'],
'testlist2': ['item2'],
'testlist3__merge_list': ['item3']
})
self.assertEqual(results, [['initial_item', 'item1', 'item3']])
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
@patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
{'item2': 'test', 'list_item': ['test2']}])
def test_merge_dict(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(['__merge_dict'], {
'testdict1__merge_dict': {
'item1': 'test',
'list_item': ['test1']
},
'testdict2__merge_dict': {
'item2': 'test',
'list_item': ['test2']
}
})
self.assertEqual(results, [
{
'item1': 'test',
'item2': 'test',
'list_item': ['test1', 'test2']
}
])
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']},
'ignore', 'suffix'])
@patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
{'item2': 'test', 'list_item': ['test2']}])
def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(['__merge_dict'], {
'testdict1__merge_dict': {
'item1': 'test',
'list_item': ['test1']
},
'testdict2__merge_dict': {
'item2': 'test',
'list_item': ['test2']
}
})
self.assertEqual(results, [
{
'initial_item': 'random value',
'item1': 'test',
'item2': 'test',
'list_item': ['test0', 'test1', 'test2']
}
])
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix'])
@patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
@patch.object(Display, 'warning')
def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display):
results = self.merge_vars_lookup.run(['__merge_non_unique'], {
'testdict1__merge_non_unique': {'item': 'value1'},
'testdict2__merge_non_unique': {'item': 'value2'}
})
self.assertTrue(mock_display.called)
self.assertEqual(results, [{'item': 'value2'}])
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix'])
@patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template):
with self.assertRaises(AnsibleError):
self.merge_vars_lookup.run(['__merge_non_unique'], {
'testdict1__merge_non_unique': {'item': 'value1'},
'testdict2__merge_non_unique': {'item': 'value2'}
})
@patch.object(AnsiblePlugin, 'set_options')
@patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
@patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
['item2', 'item3']])
def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template):
with self.assertRaises(AnsibleError):
self.merge_vars_lookup.run(['__merge_var'], {
'testlist__merge_var': {
'item1': 'test',
'list_item': ['test1']
},
'testdict__merge_var': ['item2', 'item3']
})