From 1d61541951a3ec3ecc5417bdd97ca2b1e9aca698 Mon Sep 17 00:00:00 2001 From: Vladimir Botka Date: Fri, 14 Jun 2024 21:54:58 +0200 Subject: [PATCH] Feature filter replace_keys (#8446) * Add filter replace_keys. * Update examples and integration tests. * Fix examples and copyright. * Update documentation, examples and integration tests. * Implement #8445. Add filter replace_keys * Fix documentation formatting. * Fix documentation. * Fix type(target). Formatting improved. * Instead of a dictionary, _keys_filter_target_dict returns a list * No target testing in _keys_filter_params * Interface changed _keys_filter_params(data, matching_parameter) * If there are items with equal C(before) the B(first) one will be used. * Update remove_keys. Interface changed _keys_filter_params(data, matching_parameter) * The target can't be empty also in _keys_filter_target_dict * Update plugins/filter/replace_keys.py Co-authored-by: Felix Fontein * Update plugins/filter/replace_keys.py Co-authored-by: Felix Fontein * Update plugins/filter/replace_keys.py Co-authored-by: Felix Fontein * Test attributes before and after are strings in the iteration of target. * Update plugins/filter/replace_keys.py Co-authored-by: Felix Fontein --------- Co-authored-by: Felix Fontein --- .github/BOTMETA.yml | 2 + plugins/filter/keep_keys.py | 2 +- plugins/filter/remove_keys.py | 2 +- plugins/filter/replace_keys.py | 180 ++++++++++++++++++ plugins/plugin_utils/keys_filter.py | 99 ++++++---- .../targets/filter_replace_keys/aliases | 5 + .../tasks/fn-test-replace_keys.yml | 21 ++ .../filter_replace_keys/tasks/main.yml | 7 + .../tasks/replace_keys.yml | 56 ++++++ .../targets/filter_replace_keys/vars/main.yml | 58 ++++++ 10 files changed, 397 insertions(+), 35 deletions(-) create mode 100644 plugins/filter/replace_keys.py create mode 100644 tests/integration/targets/filter_replace_keys/aliases create mode 100644 tests/integration/targets/filter_replace_keys/tasks/fn-test-replace_keys.yml create mode 100644 tests/integration/targets/filter_replace_keys/tasks/main.yml create mode 100644 tests/integration/targets/filter_replace_keys/tasks/replace_keys.yml create mode 100644 tests/integration/targets/filter_replace_keys/vars/main.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 4c6a98eaef..36d667706a 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -174,6 +174,8 @@ files: $filters/random_mac.py: {} $filters/remove_keys.py: maintainers: vbotka + $filters/replace_keys.py: + maintainers: vbotka $filters/time.py: maintainers: resmo $filters/to_days.yml: diff --git a/plugins/filter/keep_keys.py b/plugins/filter/keep_keys.py index 009e986ab2..dffccba356 100644 --- a/plugins/filter/keep_keys.py +++ b/plugins/filter/keep_keys.py @@ -110,7 +110,7 @@ def keep_keys(data, target=None, matching_parameter='equal'): """keep specific keys from dictionaries in a list""" # test parameters - _keys_filter_params(data, target, matching_parameter) + _keys_filter_params(data, matching_parameter) # test and transform target tt = _keys_filter_target_str(target, matching_parameter) diff --git a/plugins/filter/remove_keys.py b/plugins/filter/remove_keys.py index 335f82d31f..cabce14682 100644 --- a/plugins/filter/remove_keys.py +++ b/plugins/filter/remove_keys.py @@ -110,7 +110,7 @@ def remove_keys(data, target=None, matching_parameter='equal'): """remove specific keys from dictionaries in a list""" # test parameters - _keys_filter_params(data, target, matching_parameter) + _keys_filter_params(data, matching_parameter) # test and transform target tt = _keys_filter_target_str(target, matching_parameter) diff --git a/plugins/filter/replace_keys.py b/plugins/filter/replace_keys.py new file mode 100644 index 0000000000..d3b12c05d0 --- /dev/null +++ b/plugins/filter/replace_keys.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2024 Vladimir Botka +# Copyright (c) 2024 Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: replace_keys + short_description: Replace specific keys in a list of dictionaries + version_added: "9.1.0" + author: + - Vladimir Botka (@vbotka) + - Felix Fontein (@felixfontein) + description: This filter replaces specified keys in a provided list of dictionaries. + options: + _input: + description: + - A list of dictionaries. + - Top level keys must be strings. + type: list + elements: dictionary + required: true + target: + description: + - A list of dictionaries with attributes C(before) and C(after). + - The value of O(target[].after) replaces key matching O(target[].before). + type: list + elements: dictionary + required: true + suboptions: + before: + description: + - A key or key pattern to change. + - The interpretation of O(target[].before) depends on O(matching_parameter). + - For a key that matches multiple O(target[].before)s, the B(first) matching O(target[].after) will be used. + type: str + after: + description: A matching key change to. + type: str + matching_parameter: + description: Specify the matching option of target keys. + type: str + default: equal + choices: + equal: Matches keys of exactly one of the O(target[].before) items. + starts_with: Matches keys that start with one of the O(target[].before) items. + ends_with: Matches keys that end with one of the O(target[].before) items. + regex: Matches keys that match one of the regular expressions provided in O(target[].before). +''' + +EXAMPLES = ''' + l: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + + # 1) By default, replace keys that are equal any of the attributes before. + t: + - {before: k0_x0, after: a0} + - {before: k1_x1, after: a1} + r: "{{ l | community.general.replace_keys(target=t) }}" + + # 2) Replace keys that starts with any of the attributes before. + t: + - {before: k0, after: a0} + - {before: k1, after: a1} + r: "{{ l | community.general.replace_keys(target=t, matching_parameter='starts_with') }}" + + # 3) Replace keys that ends with any of the attributes before. + t: + - {before: x0, after: a0} + - {before: x1, after: a1} + r: "{{ l | community.general.replace_keys(target=t, matching_parameter='ends_with') }}" + + # 4) Replace keys that match any regex of the attributes before. + t: + - {before: "^.*0_x.*$", after: a0} + - {before: "^.*1_x.*$", after: a1} + r: "{{ l | community.general.replace_keys(target=t, matching_parameter='regex') }}" + + # The results of above examples 1-4 are all the same. + r: + - {a0: A0, a1: B0, k2_x2: [C0], k3_x3: foo} + - {a0: A1, a1: B1, k2_x2: [C1], k3_x3: bar} + + # 5) If more keys match the same attribute before the last one will be used. + t: + - {before: "^.*_x.*$", after: X} + r: "{{ l | community.general.replace_keys(target=t, matching_parameter='regex') }}" + + # gives + + r: + - X: foo + - X: bar + + # 6) If there are items with equal attribute before the first one will be used. + t: + - {before: "^.*_x.*$", after: X} + - {before: "^.*_x.*$", after: Y} + r: "{{ l | community.general.replace_keys(target=t, matching_parameter='regex') }}" + + # gives + + r: + - X: foo + - X: bar + + # 7) If there are more matches for a key the first one will be used. + l: + - {aaa1: A, bbb1: B, ccc1: C} + - {aaa2: D, bbb2: E, ccc2: F} + t: + - {before: a, after: X} + - {before: aa, after: Y} + r: "{{ l | community.general.replace_keys(target=t, matching_parameter='starts_with') }}" + + # gives + + r: + - {X: A, bbb1: B, ccc1: C} + - {X: D, bbb2: E, ccc2: F} +''' + +RETURN = ''' + _value: + description: The list of dictionaries with replaced keys. + type: list + elements: dictionary +''' + +from ansible_collections.community.general.plugins.plugin_utils.keys_filter import ( + _keys_filter_params, + _keys_filter_target_dict) + + +def replace_keys(data, target=None, matching_parameter='equal'): + """replace specific keys in a list of dictionaries""" + + # test parameters + _keys_filter_params(data, matching_parameter) + # test and transform target + tz = _keys_filter_target_dict(target, matching_parameter) + + if matching_parameter == 'equal': + def replace_key(key): + for b, a in tz: + if key == b: + return a + return key + elif matching_parameter == 'starts_with': + def replace_key(key): + for b, a in tz: + if key.startswith(b): + return a + return key + elif matching_parameter == 'ends_with': + def replace_key(key): + for b, a in tz: + if key.endswith(b): + return a + return key + elif matching_parameter == 'regex': + def replace_key(key): + for b, a in tz: + if b.match(key): + return a + return key + + return [dict((replace_key(k), v) for k, v in d.items()) for d in data] + + +class FilterModule(object): + + def filters(self): + return { + 'replace_keys': replace_keys, + } diff --git a/plugins/plugin_utils/keys_filter.py b/plugins/plugin_utils/keys_filter.py index 37b7611c50..94234a15db 100644 --- a/plugins/plugin_utils/keys_filter.py +++ b/plugins/plugin_utils/keys_filter.py @@ -13,11 +13,10 @@ from ansible.module_utils.six import string_types from ansible.module_utils.common._collections_compat import Mapping, Sequence -def _keys_filter_params(data, target, matching_parameter): +def _keys_filter_params(data, matching_parameter): """test parameters: - * data must be a list of dictionaries. All keys must be strings. - * target must be a non-empty sequence. - * matching_parameter is member of a list. + * data must be a list of dictionaries. All keys must be strings. + * matching_parameter is member of a list. """ mp = matching_parameter @@ -37,30 +36,32 @@ def _keys_filter_params(data, target, matching_parameter): msg = "Top level keys must be strings. keys: %s" raise AnsibleFilterError(msg % elem.keys()) - if not isinstance(target, Sequence): - msg = ("The target must be a string or a list. target is %s.") - raise AnsibleFilterError(msg % target) - - if len(target) == 0: - msg = ("The target can't be empty.") - raise AnsibleFilterError(msg) - if mp not in ml: - msg = ("The matching_parameter must be one of %s. matching_parameter is %s") + msg = "The matching_parameter must be one of %s. matching_parameter=%s" raise AnsibleFilterError(msg % (ml, mp)) return def _keys_filter_target_str(target, matching_parameter): - """test: - * If target is list all items are strings - * If matching_parameter=regex target is a string or list with single string - convert and return: - * tuple of unique target items, or - * tuple with single item, or - * compiled regex if matching_parameter=regex """ + Test: + * target is a non-empty string or list. + * If target is list all items are strings. + * target is a string or list with single string if matching_parameter=regex. + Convert target and return: + * tuple of unique target items, or + * tuple with single item, or + * compiled regex if matching_parameter=regex. + """ + + if not isinstance(target, Sequence): + msg = "The target must be a string or a list. target is %s." + raise AnsibleFilterError(msg % type(target)) + + if len(target) == 0: + msg = "The target can't be empty." + raise AnsibleFilterError(msg) if isinstance(target, list): for elem in target: @@ -73,15 +74,14 @@ def _keys_filter_target_str(target, matching_parameter): r = target else: if len(target) > 1: - msg = ("Single item is required in the target list if matching_parameter is regex.") + msg = "Single item is required in the target list if matching_parameter=regex." raise AnsibleFilterError(msg) else: r = target[0] try: tt = re.compile(r) except re.error: - msg = ("The target must be a valid regex if matching_parameter is regex." - " target is %s") + msg = "The target must be a valid regex if matching_parameter=regex. target is %s" raise AnsibleFilterError(msg % r) elif isinstance(target, string_types): tt = (target, ) @@ -92,17 +92,50 @@ def _keys_filter_target_str(target, matching_parameter): def _keys_filter_target_dict(target, matching_parameter): - """test: - * target is a list of dictionaries - * ... + """ + Test: + * target is a list of dictionaries with attributes 'after' and 'before'. + * Attributes 'before' must be valid regex if matching_parameter=regex. + * Otherwise, the attributes 'before' must be strings. + Convert target and return: + * iterator that aggregates attributes 'before' and 'after', or + * iterator that aggregates compiled regex of attributes 'before' and 'after' if matching_parameter=regex. """ - # TODO: Complete and use this in filter replace_keys + if not isinstance(target, list): + msg = "The target must be a list. target is %s." + raise AnsibleFilterError(msg % (target, type(target))) - if isinstance(target, list): - for elem in target: - if not isinstance(elem, Mapping): - msg = "The target items must be dictionary. %s is %s" - raise AnsibleFilterError(msg % (elem, type(elem))) + if len(target) == 0: + msg = "The target can't be empty." + raise AnsibleFilterError(msg) - return + for elem in target: + if not isinstance(elem, Mapping): + msg = "The target items must be dictionaries. %s is %s" + raise AnsibleFilterError(msg % (elem, type(elem))) + if not all(k in elem for k in ('before', 'after')): + msg = "All dictionaries in target must include attributes: after, before." + raise AnsibleFilterError(msg) + if not isinstance(elem['before'], string_types): + msg = "The attributes before must be strings. %s is %s" + raise AnsibleFilterError(msg % (elem['before'], type(elem['before']))) + if not isinstance(elem['after'], string_types): + msg = "The attributes after must be strings. %s is %s" + raise AnsibleFilterError(msg % (elem['after'], type(elem['after']))) + + before = [d['before'] for d in target] + after = [d['after'] for d in target] + + if matching_parameter == 'regex': + try: + tr = map(re.compile, before) + tz = list(zip(tr, after)) + except re.error: + msg = ("The attributes before must be valid regex if matching_parameter=regex." + " Not all items are valid regex in: %s") + raise AnsibleFilterError(msg % before) + else: + tz = list(zip(before, after)) + + return tz diff --git a/tests/integration/targets/filter_replace_keys/aliases b/tests/integration/targets/filter_replace_keys/aliases new file mode 100644 index 0000000000..12d1d6617e --- /dev/null +++ b/tests/integration/targets/filter_replace_keys/aliases @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/2 diff --git a/tests/integration/targets/filter_replace_keys/tasks/fn-test-replace_keys.yml b/tests/integration/targets/filter_replace_keys/tasks/fn-test-replace_keys.yml new file mode 100644 index 0000000000..e324376a5a --- /dev/null +++ b/tests/integration/targets/filter_replace_keys/tasks/fn-test-replace_keys.yml @@ -0,0 +1,21 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test replace keys + ansible.builtin.assert: + that: + - (rr | difference(item.result) | length) == 0 + success_msg: | + [OK] {{ item.label }} + result: + {{ rr | to_nice_yaml(indent=2) | indent(2) }} + fail_msg: | + [ERR] {{ item.label }} + result: + {{ rr | to_nice_yaml(indent=2) | indent(2) }} + quiet: "{{ quiet_test | d(true) | bool }}" + vars: + rr: "{{ item.data | + community.general.replace_keys(target=item.target, matching_parameter=item.match) }}" diff --git a/tests/integration/targets/filter_replace_keys/tasks/main.yml b/tests/integration/targets/filter_replace_keys/tasks/main.yml new file mode 100644 index 0000000000..35addaf946 --- /dev/null +++ b/tests/integration/targets/filter_replace_keys/tasks/main.yml @@ -0,0 +1,7 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test replace_keys + import_tasks: replace_keys.yml diff --git a/tests/integration/targets/filter_replace_keys/tasks/replace_keys.yml b/tests/integration/targets/filter_replace_keys/tasks/replace_keys.yml new file mode 100644 index 0000000000..a57921b81b --- /dev/null +++ b/tests/integration/targets/filter_replace_keys/tasks/replace_keys.yml @@ -0,0 +1,56 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Debug ansible_version + ansible.builtin.debug: + var: ansible_version + when: not quiet_test | d(true) | bool + tags: ansible_version + +- name: Test replace keys equal (default) + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + vars: + rr: "{{ list1 | community.general.replace_keys(target=tt) }}" + tt: + - {before: k0_x0, after: a0} + - {before: k1_x1, after: a1} + tags: equal_default + +- name: Test replace keys targets1 + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + loop: "{{ targets1 | dict2items }}" + loop_control: + label: "{{ item.key }}" + vars: + rr: "{{ list1 | community.general.replace_keys(target=item.value, matching_parameter=item.key) }}" + tags: targets1 + +- name: Test replace keys targets2 + include_tasks: + file: fn-test-replace_keys.yml + apply: + tags: targets2 + loop: "{{ targets2 }}" + loop_control: + label: "{{ item.label }}" + tags: targets2 diff --git a/tests/integration/targets/filter_replace_keys/vars/main.yml b/tests/integration/targets/filter_replace_keys/vars/main.yml new file mode 100644 index 0000000000..167e083960 --- /dev/null +++ b/tests/integration/targets/filter_replace_keys/vars/main.yml @@ -0,0 +1,58 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +list1: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + +result1: + - {a0: A0, a1: B0, k2_x2: [C0], k3_x3: foo} + - {a0: A1, a1: B1, k2_x2: [C1], k3_x3: bar} + +targets1: + equal: + - {before: k0_x0, after: a0} + - {before: k1_x1, after: a1} + starts_with: + - {before: k0, after: a0} + - {before: k1, after: a1} + ends_with: + - {before: x0, after: a0} + - {before: x1, after: a1} + regex: + - {before: "^.*0_x.*$", after: a0} + - {before: "^.*1_x.*$", after: a1} + +list2: + - {aaa1: A, bbb1: B, ccc1: C} + - {aaa2: D, bbb2: E, ccc2: F} + +targets2: + - label: If more keys match the same attribute before the last one will be used. + match: regex + target: + - {before: "^.*_x.*$", after: X} + data: "{{ list1 }}" + result: + - X: foo + - X: bar + - label: If there are items with equal attribute before the first one will be used. + match: regex + target: + - {before: "^.*_x.*$", after: X} + - {before: "^.*_x.*$", after: Y} + data: "{{ list1 }}" + result: + - X: foo + - X: bar + - label: If there are more matches for a key the first one will be used. + match: starts_with + target: + - {before: a, after: X} + - {before: aa, after: Y} + data: "{{ list2 }}" + result: + - {X: A, bbb1: B, ccc1: C} + - {X: D, bbb2: E, ccc2: F}