From eea4f4596541fb0a3fc348bf36f6208c2a408b5f Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 11 May 2021 19:27:05 +0200 Subject: [PATCH] Add dependent lookup plugin (#2164) * Add dependent lookup plugin. * Use correct YAML booleans. * Began complete rewrite. * Only match start of error msg. * Improve tests. * Work around old Jinja2 versions. * Fix metadata. * Fix filter name. --- plugins/lookup/dependent.py | 208 ++++++++++++++++++ .../targets/lookup_dependent/aliases | 2 + .../targets/lookup_dependent/tasks/main.yml | 179 +++++++++++++++ tests/unit/plugins/lookup/test_dependent.py | 44 ++++ 4 files changed, 433 insertions(+) create mode 100644 plugins/lookup/dependent.py create mode 100644 tests/integration/targets/lookup_dependent/aliases create mode 100644 tests/integration/targets/lookup_dependent/tasks/main.yml create mode 100644 tests/unit/plugins/lookup/test_dependent.py diff --git a/plugins/lookup/dependent.py b/plugins/lookup/dependent.py new file mode 100644 index 0000000000..a22a98476c --- /dev/null +++ b/plugins/lookup/dependent.py @@ -0,0 +1,208 @@ +# (c) 2015-2021, Felix Fontein +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ +name: dependent +short_description: Composes a list with nested elements of other lists or dicts which can depend on previous loop variables +version_added: 3.1.0 +description: + - "Takes the input lists and returns a list with elements that are lists, dictionaries, + or template expressions which evaluate to lists or dicts, composed of the elements of + the input evaluated lists and dictionaries." +options: + _raw: + description: + - A list where the elements are one-element dictionaries, mapping a name to a string, list, or dictionary. + The name is the index that is used in the result object. The value is iterated over as described below. + - If the value is a list, it is simply iterated over. + - If the value is a dictionary, it is iterated over and returned as if they would be processed by the + R(ansible.builtin.dict2items filter,ansible_collections.ansible.builtin.dict2items_filter). + - If the value is a string, it is evaluated as Jinja2 expressions which can access the previously chosen + elements with C(item.). The result must be a list or a dictionary. + type: list + elements: dict + required: true +""" + +EXAMPLES = """ +- name: Install/remove public keys for active admin users + ansible.posix.authorized_key: + user: "{{ item.admin.key }}" + key: "{{ lookup('file', item.key.public_key) }}" + state: "{{ 'present' if item.key.active else 'absent' }}" + when: item.admin.value.active + with_community.general.dependent: + - admin: admin_user_data + - key: admin_ssh_keys[item.admin.key] + loop_control: + # Makes the output readable, so that it doesn't contain the whole subdictionaries and lists + label: "{{ [item.admin.key, 'active' if item.key.active else 'inactive', item.key.public_key] }}" + vars: + admin_user_data: + admin1: + name: Alice + active: true + admin2: + name: Bob + active: true + admin_ssh_keys: + admin1: + - private_key: keys/private_key_admin1.pem + public_key: keys/private_key_admin1.pub + active: true + admin2: + - private_key: keys/private_key_admin2.pem + public_key: keys/private_key_admin2.pub + active: true + - private_key: keys/private_key_admin2-old.pem + public_key: keys/private_key_admin2-old.pub + active: false + +- name: Update DNS records + community.aws.route53: + zone: "{{ item.zone.key }}" + record: "{{ item.prefix.key ~ '.' if item.prefix.key else '' }}{{ item.zone.key }}" + type: "{{ item.entry.key }}" + ttl: "{{ item.entry.value.ttl | default(3600) }}" + value: "{{ item.entry.value.value }}" + state: "{{ 'absent' if (item.entry.value.absent | default(False)) else 'present' }}" + overwrite: true + loop_control: + # Makes the output readable, so that it doesn't contain the whole subdictionaries and lists + label: |- + {{ [item.zone.key, item.prefix.key, item.entry.key, + item.entry.value.ttl | default(3600), + item.entry.value.absent | default(False), item.entry.value.value] }} + with_community.general.dependent: + - zone: dns_setup + - prefix: item.zone.value + - entry: item.prefix.value + vars: + dns_setup: + example.com: + '': + A: + value: + - 1.2.3.4 + AAAA: + value: + - "2a01:1:2:3::1" + 'test._domainkey': + TXT: + ttl: 300 + value: + - '"k=rsa; t=s; p=MIGfMA..."' + example.org: + 'www': + A: + value: + - 1.2.3.4 + - 5.6.7.8 +""" + +RETURN = """ + _list: + description: + - A list composed of dictionaries whose keys are the variable names from the input list. + type: list + elements: dict + sample: + - key1: a + key2: test + - key1: a + key2: foo + - key1: b + key2: bar +""" + +from ansible.errors import AnsibleLookupError +from ansible.module_utils.common._collections_compat import Mapping, Sequence +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase +from ansible.template import Templar + + +class LookupModule(LookupBase): + def __evaluate(self, expression, templar, variables): + """Evaluate expression with templar. + + ``expression`` is the expression to evaluate. + ``variables`` are the variables to use. + """ + templar.available_variables = variables or {} + return templar.template("{0}{1}{2}".format("{{", expression, "}}"), cache=False) + + def __process(self, result, terms, index, current, templar, variables): + """Fills ``result`` list with evaluated items. + + ``result`` is a list where the resulting items are placed. + ``terms`` is the parsed list of terms + ``index`` is the current index to be processed in the list. + ``current`` is a dictionary where the first ``index`` values are filled in. + ``variables`` are the variables currently available. + """ + # If we are done, add to result list: + if index == len(terms): + result.append(current.copy()) + return + + key, expression, values = terms[index] + + if expression is not None: + # Evaluate expression in current context + vars = variables.copy() + vars['item'] = current.copy() + try: + values = self.__evaluate(expression, templar, variables=vars) + except Exception as e: + raise AnsibleLookupError( + 'Caught "{error}" while evaluating {key!r} with item == {item!r}'.format( + error=e, key=key, item=current)) + + if isinstance(values, Mapping): + for idx, val in sorted(values.items()): + current[key] = dict([('key', idx), ('value', val)]) + self.__process(result, terms, index + 1, current, templar, variables) + elif isinstance(values, Sequence): + for elt in values: + current[key] = elt + self.__process(result, terms, index + 1, current, templar, variables) + else: + raise AnsibleLookupError( + 'Did not obtain dictionary or list while evaluating {key!r} with item == {item!r}, but {type}'.format( + key=key, item=current, type=type(values))) + + def run(self, terms, variables=None, **kwargs): + """Generate list.""" + result = [] + if len(terms) > 0: + templar = Templar(loader=self._templar._loader) + data = [] + vars_so_far = set() + for index, term in enumerate(terms): + if not isinstance(term, Mapping): + raise AnsibleLookupError( + 'Parameter {index} must be a dictionary, got {type}'.format( + index=index, type=type(term))) + if len(term) != 1: + raise AnsibleLookupError( + 'Parameter {index} must be a one-element dictionary, got {count} elements'.format( + index=index, count=len(term))) + k, v = list(term.items())[0] + if k in vars_so_far: + raise AnsibleLookupError( + 'The variable {key!r} appears more than once'.format(key=k)) + vars_so_far.add(k) + if isinstance(v, string_types): + data.append((k, v, None)) + elif isinstance(v, (Sequence, Mapping)): + data.append((k, None, v)) + else: + raise AnsibleLookupError( + 'Parameter {key!r} (index {index}) must have a value of type string, dictionary or list, got type {type}'.format( + index=index, key=k, type=type(v))) + self.__process(result, data, 0, {}, templar, variables) + return result diff --git a/tests/integration/targets/lookup_dependent/aliases b/tests/integration/targets/lookup_dependent/aliases new file mode 100644 index 0000000000..45489be80c --- /dev/null +++ b/tests/integration/targets/lookup_dependent/aliases @@ -0,0 +1,2 @@ +shippable/posix/group2 +skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller diff --git a/tests/integration/targets/lookup_dependent/tasks/main.yml b/tests/integration/targets/lookup_dependent/tasks/main.yml new file mode 100644 index 0000000000..0f1b8d34fb --- /dev/null +++ b/tests/integration/targets/lookup_dependent/tasks/main.yml @@ -0,0 +1,179 @@ +--- +- name: Test 1 + set_fact: + loop_result: >- + {{ + query('community.general.dependent', + dict(key1=[1, 2]), + dict(key2='[item.key1 + 3, item.key1 + 6]'), + dict(key3='[item.key1 + item.key2 * 10]')) + }} + +- name: Check result of Test 1 + assert: + that: + - loop_result == expected_result + vars: + expected_result: + - key1: 1 + key2: 4 + key3: 41 + - key1: 1 + key2: 7 + key3: 71 + - key1: 2 + key2: 5 + key3: 52 + - key1: 2 + key2: 8 + key3: 82 + +- name: Test 2 + set_fact: + loop_result: >- + {{ query('community.general.dependent', + dict([['a', [1, 2, 3]]]), + dict([['b', '[1, 2, 3, 4] if item.a == 1 else [2, 3, 4] if item.a == 2 else [3, 4]']])) }} + # The last expression could have been `range(item.a, 5)`, but that's not supported by all Jinja2 versions used in CI + +- name: Check result of Test 2 + assert: + that: + - loop_result == expected_result + vars: + expected_result: + - a: 1 + b: 1 + - a: 1 + b: 2 + - a: 1 + b: 3 + - a: 1 + b: 4 + - a: 2 + b: 2 + - a: 2 + b: 3 + - a: 2 + b: 4 + - a: 3 + b: 3 + - a: 3 + b: 4 + +- name: Test 3 + debug: + var: item + with_community.general.dependent: + - var1: + a: + - 1 + - 2 + b: + - 3 + - 4 + - var2: 'item.var1.value' + - var3: 'dependent_lookup_test[item.var1.key ~ "_" ~ item.var2]' + loop_control: + label: "{{ [item.var1.key, item.var2, item.var3] }}" + register: dependent + vars: + dependent_lookup_test: + a_1: + - A + - B + a_2: + - C + b_3: + - D + b_4: + - E + - F + - G + +- name: Check result of Test 3 + assert: + that: + - (dependent.results | length) == 7 + - dependent.results[0].item.var1.key == "a" + - dependent.results[0].item.var2 == 1 + - dependent.results[0].item.var3 == "A" + - dependent.results[1].item.var1.key == "a" + - dependent.results[1].item.var2 == 1 + - dependent.results[1].item.var3 == "B" + - dependent.results[2].item.var1.key == "a" + - dependent.results[2].item.var2 == 2 + - dependent.results[2].item.var3 == "C" + - dependent.results[3].item.var1.key == "b" + - dependent.results[3].item.var2 == 3 + - dependent.results[3].item.var3 == "D" + - dependent.results[4].item.var1.key == "b" + - dependent.results[4].item.var2 == 4 + - dependent.results[4].item.var3 == "E" + - dependent.results[5].item.var1.key == "b" + - dependent.results[5].item.var2 == 4 + - dependent.results[5].item.var3 == "F" + - dependent.results[6].item.var1.key == "b" + - dependent.results[6].item.var2 == 4 + - dependent.results[6].item.var3 == "G" + +- name: "Test 4: template failure" + debug: + msg: "{{ item }}" + with_community.general.dependent: + - a: + - 1 + - 2 + - b: "[item.a + foo]" + ignore_errors: true + register: eval_error + +- name: Check result of Test 4 + assert: + that: + - eval_error is failed + - eval_error.msg.startswith("Caught \"'foo' is undefined\" while evaluating ") + +- name: "Test 5: same variable name reused" + debug: + msg: "{{ item }}" + with_community.general.dependent: + - a: x + - b: x + ignore_errors: true + register: eval_error + +- name: Check result of Test 5 + assert: + that: + - eval_error is failed + - eval_error.msg.startswith("Caught \"'x' is undefined\" while evaluating ") + +- name: "Test 6: multi-value dict" + debug: + msg: "{{ item }}" + with_community.general.dependent: + - a: x + b: x + ignore_errors: true + register: eval_error + +- name: Check result of Test 6 + assert: + that: + - eval_error is failed + - eval_error.msg == 'Parameter 0 must be a one-element dictionary, got 2 elements' + +- name: "Test 7: empty dict" + debug: + msg: "{{ item }}" + with_community.general.dependent: + - {} + ignore_errors: true + register: eval_error + +- name: Check result of Test 7 + assert: + that: + - eval_error is failed + - eval_error.msg == 'Parameter 0 must be a one-element dictionary, got 0 elements' diff --git a/tests/unit/plugins/lookup/test_dependent.py b/tests/unit/plugins/lookup/test_dependent.py new file mode 100644 index 0000000000..f2a31ff4b6 --- /dev/null +++ b/tests/unit/plugins/lookup/test_dependent.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# (c) 2020-2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + + +from ansible_collections.community.internal_test_tools.tests.unit.compat.unittest import TestCase +from ansible_collections.community.internal_test_tools.tests.unit.compat.mock import ( + MagicMock, +) + +from ansible.plugins.loader import lookup_loader + + +class TestLookupModule(TestCase): + def setUp(self): + templar = MagicMock() + templar._loader = None + self.lookup = lookup_loader.get("community.general.dependent", templar=templar) + + def test_empty(self): + self.assertListEqual(self.lookup.run([], None), []) + + def test_simple(self): + self.assertListEqual( + self.lookup.run( + [ + {'a': '[1, 2]'}, + {'b': '[item.a + 3, item.a + 6]'}, + {'c': '[item.a + item.b * 10]'}, + ], + {}, + ), + [ + {'a': 1, 'b': 4, 'c': 41}, + {'a': 1, 'b': 7, 'c': 71}, + {'a': 2, 'b': 5, 'c': 52}, + {'a': 2, 'b': 8, 'c': 82}, + ], + )