From f52dd194f917fa3162b158b762808aff886c27f1 Mon Sep 17 00:00:00 2001 From: Roy Lenferink Date: Sun, 26 Mar 2023 11:27:30 +0200 Subject: [PATCH] Add new merge_variables lookup plugin (#5533) * Add new merge_variables lookup plugin * Add changelog fragment * Process bot feedback * Refactor override options and add pattern_type option * Fix unit tests * Changed default pattern_type and simplified plugin based on feedback * Processed feedback for merge_variables lookup plugin * Adjust version_added. --------- Co-authored-by: Felix Fontein --- .github/BOTMETA.yml | 2 + plugins/lookup/merge_variables.py | 212 ++++++++++++++++++ .../targets/lookup_merge_variables/aliases | 6 + .../targets/lookup_merge_variables/runme.sh | 13 ++ .../targets/lookup_merge_variables/test.yml | 174 ++++++++++++++ .../lookup_merge_variables/test_with_env.yml | 44 ++++ .../targets/lookup_merge_variables/vars.yml | 34 +++ .../plugins/lookup/test_merge_variables.py | 135 +++++++++++ 8 files changed, 620 insertions(+) create mode 100644 plugins/lookup/merge_variables.py create mode 100644 tests/integration/targets/lookup_merge_variables/aliases create mode 100755 tests/integration/targets/lookup_merge_variables/runme.sh create mode 100644 tests/integration/targets/lookup_merge_variables/test.yml create mode 100644 tests/integration/targets/lookup_merge_variables/test_with_env.yml create mode 100644 tests/integration/targets/lookup_merge_variables/vars.yml create mode 100644 tests/unit/plugins/lookup/test_merge_variables.py diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 8377331216..12239527cb 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -241,6 +241,8 @@ files: $lookups/manifold.py: labels: manifold maintainers: galanoff + $lookups/merge_variables.py: + maintainers: rlenferink m-a-r-k-e $lookups/onepass: labels: onepassword maintainers: samdoran diff --git a/plugins/lookup/merge_variables.py b/plugins/lookup/merge_variables.py new file mode 100644 index 0000000000..f7b784dfe7 --- /dev/null +++ b/plugins/lookup/merge_variables.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ + author: + - Roy Lenferink (@rlenferink) + - Mark Ettema (@m-a-r-k-e) + name: merge_variables + short_description: merge variables with a certain suffix + description: + - This lookup returns the merged result of all variables in scope that match the given prefixes, suffixes, or + regular expressions, optionally. + version_added: 6.5.0 + options: + _terms: + description: + - Depending on the value of I(pattern_type), this is a list of prefixes, suffixes, or regular expressions + that will be used to match all variables that should be merged. + required: true + type: list + elements: str + pattern_type: + description: + - Change the way of searching for the specified pattern. + type: str + default: 'regex' + choices: + - prefix + - suffix + - regex + env: + - name: ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE + ini: + - section: merge_variables_lookup + key: pattern_type + initial_value: + description: + - An initial value to start with. + type: raw + override: + description: + - Return an error, print a warning or ignore it when a key will be overwritten. + - The default behavior C(error) makes the plugin fail when a key would be overwritten. + - When C(warn) and C(ignore) are used, note that it is important to know that the variables + are sorted by name before being merged. Keys for later variables in this order will overwrite + keys of the same name for variables earlier in this order. To avoid potential confusion, + better use I(override=error) whenever possible. + type: str + default: 'error' + choices: + - error + - warn + - ignore + env: + - name: ANSIBLE_MERGE_VARIABLES_OVERRIDE + ini: + - section: merge_variables_lookup + key: override +""" + +EXAMPLES = """ +# Some example variables, they can be defined anywhere as long as they are in scope +test_init_list: + - "list init item 1" + - "list init item 2" + +testa__test_list: + - "test a item 1" + +testb__test_list: + - "test b item 1" + +testa__test_dict: + ports: + - 1 + +testb__test_dict: + ports: + - 3 + + +# Merge variables that end with '__test_dict' and store the result in a variable 'example_a' +example_a: "{{ lookup('community.general.merge_variables', '__test_dict') }}" + +# The variable example_a now contains: +# ports: +# - 1 +# - 3 + + +# Merge variables that end with '__test_list', starting with an initial value and store the result +# in a variable 'example_b' +example_b: "{{ lookup('community.general.merge_variables', '__test_list', initial_value=test_init_list) }}" + +# The variable example_b now contains: +# - "list init item 1" +# - "list init item 2" +# - "test a item 1" +# - "test b item 1" +""" + +RETURN = """ + _raw: + description: In case the search matches list items, a list will be returned. In case the search matches dicts, a + dict will be returned. + type: raw + elements: raw +""" + +import re + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.utils.display import Display + +display = Display() + + +def _verify_and_get_type(variable): + if isinstance(variable, list): + return "list" + elif isinstance(variable, dict): + return "dict" + else: + raise AnsibleError("Not supported type detected, variable must be a list or a dict") + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + self.set_options(direct=kwargs) + initial_value = self.get_option("initial_value", None) + self._override = self.get_option('override', 'error') + self._pattern_type = self.get_option('pattern_type', 'regex') + + ret = [] + for term in terms: + if not isinstance(term, str): + raise AnsibleError("Non-string type '{0}' passed, only 'str' types are allowed!".format(type(term))) + + ret.append(self._merge_vars(term, initial_value, variables)) + + return ret + + def _var_matches(self, key, search_pattern): + if self._pattern_type == "prefix": + return key.startswith(search_pattern) + elif self._pattern_type == "suffix": + return key.endswith(search_pattern) + elif self._pattern_type == "regex": + matcher = re.compile(search_pattern) + return matcher.search(key) + + return False + + def _merge_vars(self, search_pattern, initial_value, variables): + display.vvv("Merge variables with {0}: {1}".format(self._pattern_type, search_pattern)) + var_merge_names = sorted([key for key in variables.keys() if self._var_matches(key, search_pattern)]) + display.vvv("The following variables will be merged: {0}".format(var_merge_names)) + + prev_var_type = None + result = None + + if initial_value is not None: + prev_var_type = _verify_and_get_type(initial_value) + result = initial_value + + for var_name in var_merge_names: + var_value = self._templar.template(variables[var_name]) # Render jinja2 templates + var_type = _verify_and_get_type(var_value) + + if prev_var_type is None: + prev_var_type = var_type + elif prev_var_type != var_type: + raise AnsibleError("Unable to merge, not all variables are of the same type") + + if result is None: + result = var_value + continue + + if var_type == "dict": + result = self._merge_dict(var_value, result, [var_name]) + else: # var_type == "list" + result += var_value + + return result + + def _merge_dict(self, src, dest, path): + for key, value in src.items(): + if isinstance(value, dict): + node = dest.setdefault(key, {}) + self._merge_dict(value, node, path + [key]) + elif isinstance(value, list) and key in dest: + dest[key] += value + else: + if (key in dest) and dest[key] != value: + msg = "The key '{0}' with value '{1}' will be overwritten with value '{2}' from '{3}.{0}'".format( + key, dest[key], value, ".".join(path)) + + if self._override == "error": + raise AnsibleError(msg) + if self._override == "warn": + display.warning(msg) + + dest[key] = value + + return dest diff --git a/tests/integration/targets/lookup_merge_variables/aliases b/tests/integration/targets/lookup_merge_variables/aliases new file mode 100644 index 0000000000..eb449a9cfe --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/aliases @@ -0,0 +1,6 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/1 +skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller diff --git a/tests/integration/targets/lookup_merge_variables/runme.sh b/tests/integration/targets/lookup_merge_variables/runme.sh new file mode 100755 index 0000000000..52a38f4a54 --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/runme.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +set -eux + +ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \ + ansible-playbook test.yml "$@" + +ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \ +ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \ + ansible-playbook test_with_env.yml "$@" diff --git a/tests/integration/targets/lookup_merge_variables/test.yml b/tests/integration/targets/lookup_merge_variables/test.yml new file mode 100644 index 0000000000..fbd884393d --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/test.yml @@ -0,0 +1,174 @@ +--- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test merge_variables lookup plugin + hosts: localhost + tasks: + - name: Include test data + include_vars: vars.yml + + # Test the default behavior + - name: Test merge list + block: + - name: Print the merged list + debug: + msg: "{{ merged_list }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list | length) == 2" + - "'item1' in merged_list" + - "'item3' in merged_list" + vars: + merged_list: "{{ lookup('community.general.merge_variables', '^.+__merge_list$') }}" + + - name: Test merge dict + block: + - name: Print the merged list + debug: + msg: "{{ merged_dict }}" + + - name: Validate that dict is complete + assert: + that: + - "'item1' in merged_dict" + - "'item2' in merged_dict" + - "'list_item' in merged_dict" + - "(merged_dict.list_item | length) == 2" + - "'test1' in (merged_dict.list_item)" + - "'test2' in (merged_dict.list_item)" + vars: + merged_dict: "{{ lookup('community.general.merge_variables', '^.+__merge_dict$') }}" + + # Test the behavior when no results are found + - name: Test merge without results + block: + - debug: + msg: "{{ not_found }}" + - name: Validate that the variable defaults to an empty list + vars: + assert: + that: + - "(not_found | default('default-used', True)) == 'default-used'" + vars: + not_found: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$') }}" + + # Test the 'pattern_type' options + - name: Test merge list (pattern_type = prefix) + block: + - name: Print the merged list + debug: + msg: "{{ merged_list }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list | length) == 4" + - "'item1' in merged_list" + - "'item2' in merged_list" + - "'item2' in merged_list" + - "'item3' in merged_list" + vars: + merged_list: "{{ lookup('community.general.merge_variables', 'testlist', pattern_type='prefix') }}" + + - name: Test merge list (pattern_type = suffix) + block: + - name: Print the merged list + debug: + msg: "{{ merged_list }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list | length) == 2" + - "'item1' in merged_list" + - "'item3' in merged_list" + vars: + merged_list: "{{ lookup('community.general.merge_variables', '__merge_list', pattern_type='suffix') }}" + + - name: Test merge list (pattern_type = regex) + block: + - name: Print the merged list + debug: + msg: "{{ merged_list }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list | length) == 3" + - "'item1' in merged_list" + - "'item2' in merged_list" + - "'item3' in merged_list" + vars: + merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}" + + # Test the 'initial_value' option + - name: Test merge without results but with initial value + block: + - name: Print the merged list + debug: + msg: "{{ not_found_initial_value }}" + + - name: Validate that the variable only contains the initial value + vars: + assert: + that: + - "(not_found_initial_value | count) == 1" + - "(not_found_initial_value | first) == 'item2'" + vars: + not_found_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$', initial_value=testlist_initial_value) }}" + + - name: Test merging a list with an initial value + block: + - name: Print the merged list + debug: + msg: "{{ merged_list_with_initial_value }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list_with_initial_value | length) == 3" + - "'item1' in merged_list_with_initial_value" + - "'item2' in merged_list_with_initial_value" + - "'item3' in merged_list_with_initial_value" + vars: + merged_list_with_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_list$', initial_value=testlist_initial_value) }}" + + # Test the 'override' options + - name: Test the 'override=warn' option + block: + - name: Print the merged list + debug: + msg: "{{ merged_with_override_warn }}" + + - name: Validate that the dict is complete and the warning is printed + assert: + that: + - "'key_to_override' in merged_with_override_warn" + - "merged_with_override_warn.key_to_override == 'Override value'" + - "'key_to_override' in lookup('file', logging_output_file)" # Check if a message is given + - "'[WARNING]' in lookup('file', logging_output_file)" # and verify that the message is a WARNING + vars: + merged_with_override_warn: "{{ lookup('community.general.merge_variables', '^.+__override_warn$', initial_value=override_warn_init, override='warn') }}" + + - name: Test the 'override=error' option + block: + - name: Validate that an override result in an error + debug: + msg: "{{ lookup('community.general.merge_variables', '^.+__override_error$', initial_value=override_error_init, override='error') }}" + ignore_errors: true # Do not stop the playbook + register: _override_error_result + + - name: Print the output + debug: + msg: "{{ _override_error_result }}" + + - name: Validate that the error is reported + assert: + that: + - "_override_error_result.failed" + - "'key_to_override' in _override_error_result.msg" diff --git a/tests/integration/targets/lookup_merge_variables/test_with_env.yml b/tests/integration/targets/lookup_merge_variables/test_with_env.yml new file mode 100644 index 0000000000..7fbb664fd6 --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/test_with_env.yml @@ -0,0 +1,44 @@ +--- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test merge_variables lookup plugin + hosts: localhost + tasks: + - name: Include test data + include_vars: vars.yml + + # Test the pattern option using the environment variable + - name: Test merge list (pattern_type = regex) + block: + - name: Print the merged list + debug: + msg: "{{ merged_list }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list | length) == 2" + - "'item1' in merged_list" + - "'item3' in merged_list" + vars: + merged_list: "{{ lookup('community.general.merge_variables', '__merge_list') }}" + + # Test whether the pattern option can be overridden + - name: Test merge list (pattern_type = suffix) + block: + - name: Print the merged list + debug: + msg: "{{ merged_list }}" + + - name: Validate that the list is complete + assert: + that: + - "(merged_list | length) == 3" + - "'item1' in merged_list" + - "'item2' in merged_list" + - "'item3' in merged_list" + vars: + merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}" diff --git a/tests/integration/targets/lookup_merge_variables/vars.yml b/tests/integration/targets/lookup_merge_variables/vars.yml new file mode 100644 index 0000000000..d1a4ace21f --- /dev/null +++ b/tests/integration/targets/lookup_merge_variables/vars.yml @@ -0,0 +1,34 @@ +--- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +testlist_initial_value: "{{ testlist2 }}" +testlist1__merge_list: + - item1 +testlist2: + - item2 +testlist3__merge_list: + - item3 + +testdict1__merge_dict: + item1: test + list_item: + - test1 +testdict2__merge_dict: + item2: test + list_item: + - test2 + +override_warn_init: + key_to_override: Initial value +override__override_warn: + key_to_override: Override value + +override_error_init: + key_to_override: Initial value +override__override_error: + key_to_override: Override value + +logging_output_file: /tmp/ansible-test-merge-variables # The Ansible log output is available in this file diff --git a/tests/unit/plugins/lookup/test_merge_variables.py b/tests/unit/plugins/lookup/test_merge_variables.py new file mode 100644 index 0000000000..5085797b3e --- /dev/null +++ b/tests/unit/plugins/lookup/test_merge_variables.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Thales Netherlands +# Copyright (c) 2021, Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible_collections.community.general.tests.unit.compat import unittest +from ansible_collections.community.general.tests.unit.compat.mock import patch +from ansible_collections.community.general.tests.unit.mock.loader import DictDataLoader + +from ansible.plugins import AnsiblePlugin +from ansible.template import Templar +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from ansible_collections.community.general.plugins.lookup import merge_variables + + +class TestMergeVariablesLookup(unittest.TestCase): + def setUp(self): + self.loader = DictDataLoader({}) + self.templar = Templar(loader=self.loader, variables={}) + self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix']) + @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']]) + def test_merge_list(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_list'], { + 'testlist1__merge_list': ['item1'], + 'testlist2': ['item2'], + 'testlist3__merge_list': ['item3'] + }) + + self.assertEqual(results, [['item1', 'item3']]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix']) + @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']]) + def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_list'], { + 'testlist1__merge_list': ['item1'], + 'testlist2': ['item2'], + 'testlist3__merge_list': ['item3'] + }) + + self.assertEqual(results, [['initial_item', 'item1', 'item3']]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix']) + @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']}, + {'item2': 'test', 'list_item': ['test2']}]) + def test_merge_dict(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_dict'], { + 'testdict1__merge_dict': { + 'item1': 'test', + 'list_item': ['test1'] + }, + 'testdict2__merge_dict': { + 'item2': 'test', + 'list_item': ['test2'] + } + }) + + self.assertEqual(results, [ + { + 'item1': 'test', + 'item2': 'test', + 'list_item': ['test1', 'test2'] + } + ]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']}, + 'ignore', 'suffix']) + @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']}, + {'item2': 'test', 'list_item': ['test2']}]) + def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template): + results = self.merge_vars_lookup.run(['__merge_dict'], { + 'testdict1__merge_dict': { + 'item1': 'test', + 'list_item': ['test1'] + }, + 'testdict2__merge_dict': { + 'item2': 'test', + 'list_item': ['test2'] + } + }) + + self.assertEqual(results, [ + { + 'initial_item': 'random value', + 'item1': 'test', + 'item2': 'test', + 'list_item': ['test0', 'test1', 'test2'] + } + ]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix']) + @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}]) + @patch.object(Display, 'warning') + def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display): + results = self.merge_vars_lookup.run(['__merge_non_unique'], { + 'testdict1__merge_non_unique': {'item': 'value1'}, + 'testdict2__merge_non_unique': {'item': 'value2'} + }) + + self.assertTrue(mock_display.called) + self.assertEqual(results, [{'item': 'value2'}]) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix']) + @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}]) + def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template): + with self.assertRaises(AnsibleError): + self.merge_vars_lookup.run(['__merge_non_unique'], { + 'testdict1__merge_non_unique': {'item': 'value1'}, + 'testdict2__merge_non_unique': {'item': 'value2'} + }) + + @patch.object(AnsiblePlugin, 'set_options') + @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix']) + @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']}, + ['item2', 'item3']]) + def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template): + with self.assertRaises(AnsibleError): + self.merge_vars_lookup.run(['__merge_var'], { + 'testlist__merge_var': { + 'item1': 'test', + 'list_item': ['test1'] + }, + 'testdict__merge_var': ['item2', 'item3'] + })