[PR #5533/f52dd194 backport][stable-6] Add new merge_variables lookup plugin (#6249)

Add new merge_variables lookup plugin (#5533)

* Add new merge_variables lookup plugin

* Add changelog fragment

* Process bot feedback

* Refactor override options and add pattern_type option

* Fix unit tests

* Changed default pattern_type and simplified plugin based on feedback

* Processed feedback for merge_variables lookup plugin

* Adjust version_added.

---------

Co-authored-by: Felix Fontein <felix@fontein.de>
(cherry picked from commit f52dd194f9)

Co-authored-by: Roy Lenferink <lenferinkroy@gmail.com>

.github/BOTMETA.yml

@@ -241,6 +241,8 @@ files:
   $lookups/manifold.py:
     labels: manifold
     maintainers: galanoff
+  $lookups/merge_variables.py:
+    maintainers: rlenferink m-a-r-k-e
   $lookups/onepass:
     labels: onepassword
     maintainers: samdoran

plugins/lookup/merge_variables.py

@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = """
+    author:
+      - Roy Lenferink (@rlenferink)
+      - Mark Ettema (@m-a-r-k-e)
+    name: merge_variables
+    short_description: merge variables with a certain suffix
+    description:
+        - This lookup returns the merged result of all variables in scope that match the given prefixes, suffixes, or
+         regular expressions, optionally.
+    version_added: 6.5.0
+    options:
+      _terms:
+        description:
+          - Depending on the value of I(pattern_type), this is a list of prefixes, suffixes, or regular expressions
+            that will be used to match all variables that should be merged.
+        required: true
+        type: list
+        elements: str
+      pattern_type:
+        description:
+          - Change the way of searching for the specified pattern.
+        type: str
+        default: 'regex'
+        choices:
+          - prefix
+          - suffix
+          - regex
+        env:
+          - name: ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE
+        ini:
+          - section: merge_variables_lookup
+            key: pattern_type
+      initial_value:
+        description:
+          - An initial value to start with.
+        type: raw
+      override:
+        description:
+          - Return an error, print a warning or ignore it when a key will be overwritten.
+          - The default behavior C(error) makes the plugin fail when a key would be overwritten.
+          - When C(warn) and C(ignore) are used, note that it is important to know that the variables
+            are sorted by name before being merged. Keys for later variables in this order will overwrite
+            keys of the same name for variables earlier in this order. To avoid potential confusion,
+            better use I(override=error) whenever possible.
+        type: str
+        default: 'error'
+        choices:
+          - error
+          - warn
+          - ignore
+        env:
+          - name: ANSIBLE_MERGE_VARIABLES_OVERRIDE
+        ini:
+          - section: merge_variables_lookup
+            key: override
+"""
+
+EXAMPLES = """
+# Some example variables, they can be defined anywhere as long as they are in scope
+test_init_list:
+  - "list init item 1"
+  - "list init item 2"
+
+testa__test_list:
+  - "test a item 1"
+
+testb__test_list:
+  - "test b item 1"
+
+testa__test_dict:
+  ports:
+    - 1
+
+testb__test_dict:
+  ports:
+    - 3
+
+
+# Merge variables that end with '__test_dict' and store the result in a variable 'example_a'
+example_a: "{{ lookup('community.general.merge_variables', '__test_dict') }}"
+
+# The variable example_a now contains:
+# ports:
+#   - 1
+#   - 3
+
+
+# Merge variables that end with '__test_list', starting with an initial value and store the result
+# in a variable 'example_b'
+example_b: "{{ lookup('community.general.merge_variables', '__test_list', initial_value=test_init_list) }}"
+
+# The variable example_b now contains:
+#   - "list init item 1"
+#   - "list init item 2"
+#   - "test a item 1"
+#   - "test b item 1"
+"""
+
+RETURN = """
+  _raw:
+    description: In case the search matches list items, a list will be returned. In case the search matches dicts, a
+     dict will be returned.
+    type: raw
+    elements: raw
+"""
+
+import re
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+from ansible.utils.display import Display
+
+display = Display()
+
+
+def _verify_and_get_type(variable):
+    if isinstance(variable, list):
+        return "list"
+    elif isinstance(variable, dict):
+        return "dict"
+    else:
+        raise AnsibleError("Not supported type detected, variable must be a list or a dict")
+
+
+class LookupModule(LookupBase):
+
+    def run(self, terms, variables=None, **kwargs):
+        self.set_options(direct=kwargs)
+        initial_value = self.get_option("initial_value", None)
+        self._override = self.get_option('override', 'error')
+        self._pattern_type = self.get_option('pattern_type', 'regex')
+
+        ret = []
+        for term in terms:
+            if not isinstance(term, str):
+                raise AnsibleError("Non-string type '{0}' passed, only 'str' types are allowed!".format(type(term)))
+
+            ret.append(self._merge_vars(term, initial_value, variables))
+
+        return ret
+
+    def _var_matches(self, key, search_pattern):
+        if self._pattern_type == "prefix":
+            return key.startswith(search_pattern)
+        elif self._pattern_type == "suffix":
+            return key.endswith(search_pattern)
+        elif self._pattern_type == "regex":
+            matcher = re.compile(search_pattern)
+            return matcher.search(key)
+
+        return False
+
+    def _merge_vars(self, search_pattern, initial_value, variables):
+        display.vvv("Merge variables with {0}: {1}".format(self._pattern_type, search_pattern))
+        var_merge_names = sorted([key for key in variables.keys() if self._var_matches(key, search_pattern)])
+        display.vvv("The following variables will be merged: {0}".format(var_merge_names))
+
+        prev_var_type = None
+        result = None
+
+        if initial_value is not None:
+            prev_var_type = _verify_and_get_type(initial_value)
+            result = initial_value
+
+        for var_name in var_merge_names:
+            var_value = self._templar.template(variables[var_name])  # Render jinja2 templates
+            var_type = _verify_and_get_type(var_value)
+
+            if prev_var_type is None:
+                prev_var_type = var_type
+            elif prev_var_type != var_type:
+                raise AnsibleError("Unable to merge, not all variables are of the same type")
+
+            if result is None:
+                result = var_value
+                continue
+
+            if var_type == "dict":
+                result = self._merge_dict(var_value, result, [var_name])
+            else:  # var_type == "list"
+                result += var_value
+
+        return result
+
+    def _merge_dict(self, src, dest, path):
+        for key, value in src.items():
+            if isinstance(value, dict):
+                node = dest.setdefault(key, {})
+                self._merge_dict(value, node, path + [key])
+            elif isinstance(value, list) and key in dest:
+                dest[key] += value
+            else:
+                if (key in dest) and dest[key] != value:
+                    msg = "The key '{0}' with value '{1}' will be overwritten with value '{2}' from '{3}.{0}'".format(
+                        key, dest[key], value, ".".join(path))
+
+                    if self._override == "error":
+                        raise AnsibleError(msg)
+                    if self._override == "warn":
+                        display.warning(msg)
+
+                dest[key] = value
+
+        return dest

tests/integration/targets/lookup_merge_variables/aliases

@@ -0,0 +1,6 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/1
+skip/python2.6  # lookups are controller only, and we no longer support Python 2.6 on the controller

tests/integration/targets/lookup_merge_variables/runme.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+set -eux
+
+ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
+    ansible-playbook test.yml "$@"
+
+ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
+ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \
+    ansible-playbook test_with_env.yml "$@"

tests/integration/targets/lookup_merge_variables/test.yml

@@ -0,0 +1,174 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test merge_variables lookup plugin
+  hosts: localhost
+  tasks:
+    - name: Include test data
+      include_vars: vars.yml
+
+    # Test the default behavior
+    - name: Test merge list
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 2"
+              - "'item1' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '^.+__merge_list$') }}"
+
+    - name: Test merge dict
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Validate that dict is complete
+          assert:
+            that:
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "(merged_dict.list_item | length) == 2"
+              - "'test1' in (merged_dict.list_item)"
+              - "'test2' in (merged_dict.list_item)"
+      vars:
+        merged_dict: "{{ lookup('community.general.merge_variables', '^.+__merge_dict$') }}"
+
+    # Test the behavior when no results are found
+    - name: Test merge without results
+      block:
+        - debug:
+            msg: "{{ not_found }}"
+        - name: Validate that the variable defaults to an empty list
+          vars:
+          assert:
+            that:
+              - "(not_found | default('default-used', True)) == 'default-used'"
+      vars:
+        not_found: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$') }}"
+
+    # Test the 'pattern_type' options
+    - name: Test merge list (pattern_type = prefix)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 4"
+              - "'item1' in merged_list"
+              - "'item2' in merged_list"
+              - "'item2' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', 'testlist', pattern_type='prefix') }}"
+
+    - name: Test merge list (pattern_type = suffix)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 2"
+              - "'item1' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '__merge_list', pattern_type='suffix') }}"
+
+    - name: Test merge list (pattern_type = regex)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 3"
+              - "'item1' in merged_list"
+              - "'item2' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}"
+
+    # Test the 'initial_value' option
+    - name: Test merge without results but with initial value
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ not_found_initial_value }}"
+
+        - name: Validate that the variable only contains the initial value
+          vars:
+          assert:
+            that:
+              - "(not_found_initial_value | count) == 1"
+              - "(not_found_initial_value | first) == 'item2'"
+      vars:
+        not_found_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$', initial_value=testlist_initial_value) }}"
+
+    - name: Test merging a list with an initial value
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list_with_initial_value }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list_with_initial_value | length) == 3"
+              - "'item1' in merged_list_with_initial_value"
+              - "'item2' in merged_list_with_initial_value"
+              - "'item3' in merged_list_with_initial_value"
+      vars:
+        merged_list_with_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_list$', initial_value=testlist_initial_value) }}"
+
+    # Test the 'override' options
+    - name: Test the 'override=warn' option
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_with_override_warn }}"
+
+        - name: Validate that the dict is complete and the warning is printed
+          assert:
+            that:
+              - "'key_to_override' in merged_with_override_warn"
+              - "merged_with_override_warn.key_to_override == 'Override value'"
+              - "'key_to_override' in lookup('file', logging_output_file)"  # Check if a message is given
+              - "'[WARNING]' in lookup('file', logging_output_file)"        # and verify that the message is a WARNING
+      vars:
+        merged_with_override_warn: "{{ lookup('community.general.merge_variables', '^.+__override_warn$', initial_value=override_warn_init, override='warn') }}"
+
+    - name: Test the 'override=error' option
+      block:
+        - name: Validate that an override result in an error
+          debug:
+            msg: "{{ lookup('community.general.merge_variables', '^.+__override_error$', initial_value=override_error_init, override='error') }}"
+          ignore_errors: true  # Do not stop the playbook
+          register: _override_error_result
+
+        - name: Print the output
+          debug:
+            msg: "{{ _override_error_result }}"
+
+        - name: Validate that the error is reported
+          assert:
+            that:
+              - "_override_error_result.failed"
+              - "'key_to_override' in _override_error_result.msg"

tests/integration/targets/lookup_merge_variables/test_with_env.yml

@@ -0,0 +1,44 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test merge_variables lookup plugin
+  hosts: localhost
+  tasks:
+    - name: Include test data
+      include_vars: vars.yml
+
+    # Test the pattern option using the environment variable
+    - name: Test merge list (pattern_type = regex)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 2"
+              - "'item1' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '__merge_list') }}"
+
+    # Test whether the pattern option can be overridden
+    - name: Test merge list (pattern_type = suffix)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 3"
+              - "'item1' in merged_list"
+              - "'item2' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}"

tests/integration/targets/lookup_merge_variables/vars.yml

@@ -0,0 +1,34 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+testlist_initial_value: "{{ testlist2 }}"
+testlist1__merge_list:
+  - item1
+testlist2:
+  - item2
+testlist3__merge_list:
+  - item3
+
+testdict1__merge_dict:
+  item1: test
+  list_item:
+    - test1
+testdict2__merge_dict:
+  item2: test
+  list_item:
+    - test2
+
+override_warn_init:
+  key_to_override: Initial value
+override__override_warn:
+  key_to_override: Override value
+
+override_error_init:
+  key_to_override: Initial value
+override__override_error:
+  key_to_override: Override value
+
+logging_output_file: /tmp/ansible-test-merge-variables  # The Ansible log output is available in this file

tests/unit/plugins/lookup/test_merge_variables.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible_collections.community.general.tests.unit.compat import unittest
+from ansible_collections.community.general.tests.unit.compat.mock import patch
+from ansible_collections.community.general.tests.unit.mock.loader import DictDataLoader
+
+from ansible.plugins import AnsiblePlugin
+from ansible.template import Templar
+from ansible.errors import AnsibleError
+from ansible.utils.display import Display
+from ansible_collections.community.general.plugins.lookup import merge_variables
+
+
+class TestMergeVariablesLookup(unittest.TestCase):
+    def setUp(self):
+        self.loader = DictDataLoader({})
+        self.templar = Templar(loader=self.loader, variables={})
+        self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar)
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
+    def test_merge_list(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_list'], {
+            'testlist1__merge_list': ['item1'],
+            'testlist2': ['item2'],
+            'testlist3__merge_list': ['item3']
+        })
+
+        self.assertEqual(results, [['item1', 'item3']])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
+    def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_list'], {
+            'testlist1__merge_list': ['item1'],
+            'testlist2': ['item2'],
+            'testlist3__merge_list': ['item3']
+        })
+
+        self.assertEqual(results, [['initial_item', 'item1', 'item3']])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
+                                                    {'item2': 'test', 'list_item': ['test2']}])
+    def test_merge_dict(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_dict'], {
+            'testdict1__merge_dict': {
+                'item1': 'test',
+                'list_item': ['test1']
+            },
+            'testdict2__merge_dict': {
+                'item2': 'test',
+                'list_item': ['test2']
+            }
+        })
+
+        self.assertEqual(results, [
+            {
+                'item1': 'test',
+                'item2': 'test',
+                'list_item': ['test1', 'test2']
+            }
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']},
+                                                            'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
+                                                    {'item2': 'test', 'list_item': ['test2']}])
+    def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_dict'], {
+            'testdict1__merge_dict': {
+                'item1': 'test',
+                'list_item': ['test1']
+            },
+            'testdict2__merge_dict': {
+                'item2': 'test',
+                'list_item': ['test2']
+            }
+        })
+
+        self.assertEqual(results, [
+            {
+                'initial_item': 'random value',
+                'item1': 'test',
+                'item2': 'test',
+                'list_item': ['test0', 'test1', 'test2']
+            }
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
+    @patch.object(Display, 'warning')
+    def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display):
+        results = self.merge_vars_lookup.run(['__merge_non_unique'], {
+            'testdict1__merge_non_unique': {'item': 'value1'},
+            'testdict2__merge_non_unique': {'item': 'value2'}
+        })
+
+        self.assertTrue(mock_display.called)
+        self.assertEqual(results, [{'item': 'value2'}])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
+    def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template):
+        with self.assertRaises(AnsibleError):
+            self.merge_vars_lookup.run(['__merge_non_unique'], {
+                'testdict1__merge_non_unique': {'item': 'value1'},
+                'testdict2__merge_non_unique': {'item': 'value2'}
+            })
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
+                                                    ['item2', 'item3']])
+    def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template):
+        with self.assertRaises(AnsibleError):
+            self.merge_vars_lookup.run(['__merge_var'], {
+                'testlist__merge_var': {
+                    'item1': 'test',
+                    'list_item': ['test1']
+                },
+                'testdict__merge_var': ['item2', 'item3']
+            })