Add dependent lookup plugin (#2164) (#2490)

* Add dependent lookup plugin.

* Use correct YAML booleans.

* Began complete rewrite.

* Only match start of error msg.

* Improve tests.

* Work around old Jinja2 versions.

* Fix metadata.

* Fix filter name.

(cherry picked from commit eea4f45965)

Co-authored-by: Felix Fontein <felix@fontein.de>

plugins/lookup/dependent.py

@@ -0,0 +1,208 @@
+# (c) 2015-2021, Felix Fontein <felix@fontein.de>
+# (c) 2018 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = """
+name: dependent
+short_description: Composes a list with nested elements of other lists or dicts which can depend on previous loop variables
+version_added: 3.1.0
+description:
+  - "Takes the input lists and returns a list with elements that are lists, dictionaries,
+     or template expressions which evaluate to lists or dicts, composed of the elements of
+     the input evaluated lists and dictionaries."
+options:
+  _raw:
+    description:
+      - A list where the elements are one-element dictionaries, mapping a name to a string, list, or dictionary.
+        The name is the index that is used in the result object. The value is iterated over as described below.
+      - If the value is a list, it is simply iterated over.
+      - If the value is a dictionary, it is iterated over and returned as if they would be processed by the
+        R(ansible.builtin.dict2items filter,ansible_collections.ansible.builtin.dict2items_filter).
+      - If the value is a string, it is evaluated as Jinja2 expressions which can access the previously chosen
+        elements with C(item.<index_name>). The result must be a list or a dictionary.
+    type: list
+    elements: dict
+    required: true
+"""
+
+EXAMPLES = """
+- name: Install/remove public keys for active admin users
+  ansible.posix.authorized_key:
+    user: "{{ item.admin.key }}"
+    key: "{{ lookup('file', item.key.public_key) }}"
+    state: "{{ 'present' if item.key.active else 'absent' }}"
+  when: item.admin.value.active
+  with_community.general.dependent:
+    - admin: admin_user_data
+    - key: admin_ssh_keys[item.admin.key]
+  loop_control:
+    # Makes the output readable, so that it doesn't contain the whole subdictionaries and lists
+    label: "{{ [item.admin.key, 'active' if item.key.active else 'inactive', item.key.public_key] }}"
+  vars:
+    admin_user_data:
+      admin1:
+        name: Alice
+        active: true
+      admin2:
+        name: Bob
+        active: true
+    admin_ssh_keys:
+      admin1:
+        - private_key: keys/private_key_admin1.pem
+          public_key: keys/private_key_admin1.pub
+          active: true
+      admin2:
+        - private_key: keys/private_key_admin2.pem
+          public_key: keys/private_key_admin2.pub
+          active: true
+        - private_key: keys/private_key_admin2-old.pem
+          public_key: keys/private_key_admin2-old.pub
+          active: false
+
+- name: Update DNS records
+  community.aws.route53:
+    zone: "{{ item.zone.key }}"
+    record: "{{ item.prefix.key ~ '.' if item.prefix.key else '' }}{{ item.zone.key }}"
+    type: "{{ item.entry.key }}"
+    ttl: "{{ item.entry.value.ttl | default(3600) }}"
+    value: "{{ item.entry.value.value }}"
+    state: "{{ 'absent' if (item.entry.value.absent | default(False)) else 'present' }}"
+    overwrite: true
+  loop_control:
+    # Makes the output readable, so that it doesn't contain the whole subdictionaries and lists
+    label: |-
+        {{ [item.zone.key, item.prefix.key, item.entry.key,
+            item.entry.value.ttl | default(3600),
+            item.entry.value.absent | default(False), item.entry.value.value] }}
+  with_community.general.dependent:
+    - zone: dns_setup
+    - prefix: item.zone.value
+    - entry: item.prefix.value
+  vars:
+    dns_setup:
+      example.com:
+        '':
+          A:
+            value:
+            - 1.2.3.4
+          AAAA:
+            value:
+            - "2a01:1:2:3::1"
+        'test._domainkey':
+          TXT:
+            ttl: 300
+            value:
+            - '"k=rsa; t=s; p=MIGfMA..."'
+      example.org:
+        'www':
+          A:
+            value:
+            - 1.2.3.4
+            - 5.6.7.8
+"""
+
+RETURN = """
+  _list:
+    description:
+      - A list composed of dictionaries whose keys are the variable names from the input list.
+    type: list
+    elements: dict
+    sample:
+      - key1: a
+        key2: test
+      - key1: a
+        key2: foo
+      - key1: b
+        key2: bar
+"""
+
+from ansible.errors import AnsibleLookupError
+from ansible.module_utils.common._collections_compat import Mapping, Sequence
+from ansible.module_utils.six import string_types
+from ansible.plugins.lookup import LookupBase
+from ansible.template import Templar
+
+
+class LookupModule(LookupBase):
+    def __evaluate(self, expression, templar, variables):
+        """Evaluate expression with templar.
+
+        ``expression`` is the expression to evaluate.
+        ``variables`` are the variables to use.
+        """
+        templar.available_variables = variables or {}
+        return templar.template("{0}{1}{2}".format("{{", expression, "}}"), cache=False)
+
+    def __process(self, result, terms, index, current, templar, variables):
+        """Fills ``result`` list with evaluated items.
+
+        ``result`` is a list where the resulting items are placed.
+        ``terms`` is the parsed list of terms
+        ``index`` is the current index to be processed in the list.
+        ``current`` is a dictionary where the first ``index`` values are filled in.
+        ``variables`` are the variables currently available.
+        """
+        # If we are done, add to result list:
+        if index == len(terms):
+            result.append(current.copy())
+            return
+
+        key, expression, values = terms[index]
+
+        if expression is not None:
+            # Evaluate expression in current context
+            vars = variables.copy()
+            vars['item'] = current.copy()
+            try:
+                values = self.__evaluate(expression, templar, variables=vars)
+            except Exception as e:
+                raise AnsibleLookupError(
+                    'Caught "{error}" while evaluating {key!r} with item == {item!r}'.format(
+                        error=e, key=key, item=current))
+
+        if isinstance(values, Mapping):
+            for idx, val in sorted(values.items()):
+                current[key] = dict([('key', idx), ('value', val)])
+                self.__process(result, terms, index + 1, current, templar, variables)
+        elif isinstance(values, Sequence):
+            for elt in values:
+                current[key] = elt
+                self.__process(result, terms, index + 1, current, templar, variables)
+        else:
+            raise AnsibleLookupError(
+                'Did not obtain dictionary or list while evaluating {key!r} with item == {item!r}, but {type}'.format(
+                    key=key, item=current, type=type(values)))
+
+    def run(self, terms, variables=None, **kwargs):
+        """Generate list."""
+        result = []
+        if len(terms) > 0:
+            templar = Templar(loader=self._templar._loader)
+            data = []
+            vars_so_far = set()
+            for index, term in enumerate(terms):
+                if not isinstance(term, Mapping):
+                    raise AnsibleLookupError(
+                        'Parameter {index} must be a dictionary, got {type}'.format(
+                            index=index, type=type(term)))
+                if len(term) != 1:
+                    raise AnsibleLookupError(
+                        'Parameter {index} must be a one-element dictionary, got {count} elements'.format(
+                            index=index, count=len(term)))
+                k, v = list(term.items())[0]
+                if k in vars_so_far:
+                    raise AnsibleLookupError(
+                        'The variable {key!r} appears more than once'.format(key=k))
+                vars_so_far.add(k)
+                if isinstance(v, string_types):
+                    data.append((k, v, None))
+                elif isinstance(v, (Sequence, Mapping)):
+                    data.append((k, None, v))
+                else:
+                    raise AnsibleLookupError(
+                        'Parameter {key!r} (index {index}) must have a value of type string, dictionary or list, got type {type}'.format(
+                            index=index, key=k, type=type(v)))
+            self.__process(result, data, 0, {}, templar, variables)
+        return result

tests/integration/targets/lookup_dependent/aliases

@@ -0,0 +1,2 @@
+shippable/posix/group2
+skip/python2.6  # lookups are controller only, and we no longer support Python 2.6 on the controller

tests/integration/targets/lookup_dependent/tasks/main.yml

@@ -0,0 +1,179 @@
+---
+- name: Test 1
+  set_fact:
+    loop_result: >-
+      {{
+        query('community.general.dependent',
+          dict(key1=[1, 2]),
+          dict(key2='[item.key1 + 3, item.key1 + 6]'),
+          dict(key3='[item.key1 + item.key2 * 10]'))
+      }}
+
+- name: Check result of Test 1
+  assert:
+    that:
+      - loop_result == expected_result
+  vars:
+    expected_result:
+      - key1: 1
+        key2: 4
+        key3: 41
+      - key1: 1
+        key2: 7
+        key3: 71
+      - key1: 2
+        key2: 5
+        key3: 52
+      - key1: 2
+        key2: 8
+        key3: 82
+
+- name: Test 2
+  set_fact:
+    loop_result: >-
+      {{ query('community.general.dependent',
+               dict([['a', [1, 2, 3]]]),
+               dict([['b', '[1, 2, 3, 4] if item.a == 1 else [2, 3, 4] if item.a == 2 else [3, 4]']])) }}
+    # The last expression could have been `range(item.a, 5)`, but that's not supported by all Jinja2 versions used in CI
+
+- name: Check result of Test 2
+  assert:
+    that:
+      - loop_result == expected_result
+  vars:
+    expected_result:
+      - a: 1
+        b: 1
+      - a: 1
+        b: 2
+      - a: 1
+        b: 3
+      - a: 1
+        b: 4
+      - a: 2
+        b: 2
+      - a: 2
+        b: 3
+      - a: 2
+        b: 4
+      - a: 3
+        b: 3
+      - a: 3
+        b: 4
+
+- name: Test 3
+  debug:
+    var: item
+  with_community.general.dependent:
+    - var1:
+        a:
+          - 1
+          - 2
+        b:
+          - 3
+          - 4
+    - var2: 'item.var1.value'
+    - var3: 'dependent_lookup_test[item.var1.key ~ "_" ~ item.var2]'
+  loop_control:
+    label: "{{ [item.var1.key, item.var2, item.var3] }}"
+  register: dependent
+  vars:
+    dependent_lookup_test:
+      a_1:
+        - A
+        - B
+      a_2:
+        - C
+      b_3:
+        - D
+      b_4:
+        - E
+        - F
+        - G
+
+- name: Check result of Test 3
+  assert:
+    that:
+      - (dependent.results | length) == 7
+      - dependent.results[0].item.var1.key == "a"
+      - dependent.results[0].item.var2 == 1
+      - dependent.results[0].item.var3 == "A"
+      - dependent.results[1].item.var1.key == "a"
+      - dependent.results[1].item.var2 == 1
+      - dependent.results[1].item.var3 == "B"
+      - dependent.results[2].item.var1.key == "a"
+      - dependent.results[2].item.var2 == 2
+      - dependent.results[2].item.var3 == "C"
+      - dependent.results[3].item.var1.key == "b"
+      - dependent.results[3].item.var2 == 3
+      - dependent.results[3].item.var3 == "D"
+      - dependent.results[4].item.var1.key == "b"
+      - dependent.results[4].item.var2 == 4
+      - dependent.results[4].item.var3 == "E"
+      - dependent.results[5].item.var1.key == "b"
+      - dependent.results[5].item.var2 == 4
+      - dependent.results[5].item.var3 == "F"
+      - dependent.results[6].item.var1.key == "b"
+      - dependent.results[6].item.var2 == 4
+      - dependent.results[6].item.var3 == "G"
+
+- name: "Test 4: template failure"
+  debug:
+    msg: "{{ item }}"
+  with_community.general.dependent:
+    - a:
+        - 1
+        - 2
+    - b: "[item.a + foo]"
+  ignore_errors: true
+  register: eval_error
+
+- name: Check result of Test 4
+  assert:
+    that:
+      - eval_error is failed
+      - eval_error.msg.startswith("Caught \"'foo' is undefined\" while evaluating ")
+
+- name: "Test 5: same variable name reused"
+  debug:
+    msg: "{{ item }}"
+  with_community.general.dependent:
+    - a: x
+    - b: x
+  ignore_errors: true
+  register: eval_error
+
+- name: Check result of Test 5
+  assert:
+    that:
+      - eval_error is failed
+      - eval_error.msg.startswith("Caught \"'x' is undefined\" while evaluating ")
+
+- name: "Test 6: multi-value dict"
+  debug:
+    msg: "{{ item }}"
+  with_community.general.dependent:
+    - a: x
+      b: x
+  ignore_errors: true
+  register: eval_error
+
+- name: Check result of Test 6
+  assert:
+    that:
+      - eval_error is failed
+      - eval_error.msg == 'Parameter 0 must be a one-element dictionary, got 2 elements'
+
+- name: "Test 7: empty dict"
+  debug:
+    msg: "{{ item }}"
+  with_community.general.dependent:
+    - {}
+  ignore_errors: true
+  register: eval_error
+
+- name: Check result of Test 7
+  assert:
+    that:
+      - eval_error is failed
+      - eval_error.msg == 'Parameter 0 must be a one-element dictionary, got 0 elements'

tests/unit/plugins/lookup/test_dependent.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# (c) 2020-2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+
+from ansible_collections.community.internal_test_tools.tests.unit.compat.unittest import TestCase
+from ansible_collections.community.internal_test_tools.tests.unit.compat.mock import (
+    MagicMock,
+)
+
+from ansible.plugins.loader import lookup_loader
+
+
+class TestLookupModule(TestCase):
+    def setUp(self):
+        templar = MagicMock()
+        templar._loader = None
+        self.lookup = lookup_loader.get("community.general.dependent", templar=templar)
+
+    def test_empty(self):
+        self.assertListEqual(self.lookup.run([], None), [])
+
+    def test_simple(self):
+        self.assertListEqual(
+            self.lookup.run(
+                [
+                    {'a': '[1, 2]'},
+                    {'b': '[item.a + 3, item.a + 6]'},
+                    {'c': '[item.a + item.b * 10]'},
+                ],
+                {},
+            ),
+            [
+                {'a': 1, 'b': 4, 'c': 41},
+                {'a': 1, 'b': 7, 'c': 71},
+                {'a': 2, 'b': 5, 'c': 52},
+                {'a': 2, 'b': 8, 'c': 82},
+            ],
+        )