From 16d5d5fc57774e053a8f68a931ac62525266f978 Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:14:20 +0200 Subject: [PATCH] [PR #8443/06f13e79 backport][stable-9] Feature filter remove_keys (#8473) Feature filter remove_keys (#8443) * Add filter remove_keys. * Add filter remove_keys integration test, fragment, and maintainer. * Update with plugins/plugin_utils/keys_filter.py * Update according PR #8456 * Update maintainers. * Fix typo in return doc. * Remove local keys_filter.py. Then rebase. * Add local keys_filter.py * Update plugins/filter/remove_keys.py Co-authored-by: Felix Fontein * Update plugins/filter/remove_keys.py Co-authored-by: Felix Fontein --------- Co-authored-by: Felix Fontein (cherry picked from commit 06f13e79b1bfac4cb33cce300856618582f238ef) Co-authored-by: Vladimir Botka --- .github/BOTMETA.yml | 2 + plugins/filter/remove_keys.py | 138 ++++++++++++++++++ .../targets/filter_remove_keys/aliases | 5 + .../targets/filter_remove_keys/tasks/main.yml | 7 + .../filter_remove_keys/tasks/remove_keys.yml | 79 ++++++++++ .../targets/filter_remove_keys/vars/main.yml | 33 +++++ 6 files changed, 264 insertions(+) create mode 100644 plugins/filter/remove_keys.py create mode 100644 tests/integration/targets/filter_remove_keys/aliases create mode 100644 tests/integration/targets/filter_remove_keys/tasks/main.yml create mode 100644 tests/integration/targets/filter_remove_keys/tasks/remove_keys.yml create mode 100644 tests/integration/targets/filter_remove_keys/vars/main.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 4ffd0783b7..ade18c0e33 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -172,6 +172,8 @@ files: $filters/lists_union.yml: maintainers: cfiehe $filters/random_mac.py: {} + $filters/remove_keys.py: + maintainers: vbotka $filters/time.py: maintainers: resmo $filters/to_days.yml: diff --git a/plugins/filter/remove_keys.py b/plugins/filter/remove_keys.py new file mode 100644 index 0000000000..335f82d31f --- /dev/null +++ b/plugins/filter/remove_keys.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2024 Vladimir Botka +# Copyright (c) 2024 Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: remove_keys + short_description: Remove specific keys from dictionaries in a list + version_added: "9.1.0" + author: + - Vladimir Botka (@vbotka) + - Felix Fontein (@felixfontein) + description: This filter removes only specified keys from a provided list of dictionaries. + options: + _input: + description: + - A list of dictionaries. + - Top level keys must be strings. + type: list + elements: dictionary + required: true + target: + description: + - A single key or key pattern to remove, or a list of keys or keys patterns to remove. + - If O(matching_parameter=regex) there must be exactly one pattern provided. + type: raw + required: true + matching_parameter: + description: Specify the matching option of target keys. + type: str + default: equal + choices: + equal: Matches keys of exactly one of the O(target) items. + starts_with: Matches keys that start with one of the O(target) items. + ends_with: Matches keys that end with one of the O(target) items. + regex: + - Matches keys that match the regular expresion provided in O(target). + - In this case, O(target) must be a regex string or a list with single regex string. +''' + +EXAMPLES = ''' + l: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + + # 1) By default match keys that equal any of the items in the target. + t: [k0_x0, k1_x1] + r: "{{ l | community.general.remove_keys(target=t) }}" + + # 2) Match keys that start with any of the items in the target. + t: [k0, k1] + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='starts_with') }}" + + # 3) Match keys that end with any of the items in target. + t: [x0, x1] + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='ends_with') }}" + + # 4) Match keys by the regex. + t: ['^.*[01]_x.*$'] + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='regex') }}" + + # 5) Match keys by the regex. + t: '^.*[01]_x.*$' + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='regex') }}" + + # The results of above examples 1-5 are all the same. + r: + - {k2_x2: [C0], k3_x3: foo} + - {k2_x2: [C1], k3_x3: bar} + + # 6) By default match keys that equal the target. + t: k0_x0 + r: "{{ l | community.general.remove_keys(target=t) }}" + + # 7) Match keys that start with the target. + t: k0 + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='starts_with') }}" + + # 8) Match keys that end with the target. + t: x0 + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='ends_with') }}" + + # 9) Match keys by the regex. + t: '^.*0_x.*$' + r: "{{ l | community.general.remove_keys(target=t, matching_parameter='regex') }}" + + # The results of above examples 6-9 are all the same. + r: + - {k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k1_x1: B1, k2_x2: [C1], k3_x3: bar} +''' + +RETURN = ''' + _value: + description: The list of dictionaries with selected keys removed. + type: list + elements: dictionary +''' + +from ansible_collections.community.general.plugins.plugin_utils.keys_filter import ( + _keys_filter_params, + _keys_filter_target_str) + + +def remove_keys(data, target=None, matching_parameter='equal'): + """remove specific keys from dictionaries in a list""" + + # test parameters + _keys_filter_params(data, target, matching_parameter) + # test and transform target + tt = _keys_filter_target_str(target, matching_parameter) + + if matching_parameter == 'equal': + def keep_key(key): + return key not in tt + elif matching_parameter == 'starts_with': + def keep_key(key): + return not key.startswith(tt) + elif matching_parameter == 'ends_with': + def keep_key(key): + return not key.endswith(tt) + elif matching_parameter == 'regex': + def keep_key(key): + return tt.match(key) is None + + return [dict((k, v) for k, v in d.items() if keep_key(k)) for d in data] + + +class FilterModule(object): + + def filters(self): + return { + 'remove_keys': remove_keys, + } diff --git a/tests/integration/targets/filter_remove_keys/aliases b/tests/integration/targets/filter_remove_keys/aliases new file mode 100644 index 0000000000..12d1d6617e --- /dev/null +++ b/tests/integration/targets/filter_remove_keys/aliases @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/2 diff --git a/tests/integration/targets/filter_remove_keys/tasks/main.yml b/tests/integration/targets/filter_remove_keys/tasks/main.yml new file mode 100644 index 0000000000..d4215d8c59 --- /dev/null +++ b/tests/integration/targets/filter_remove_keys/tasks/main.yml @@ -0,0 +1,7 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test remove_keys + import_tasks: remove_keys.yml diff --git a/tests/integration/targets/filter_remove_keys/tasks/remove_keys.yml b/tests/integration/targets/filter_remove_keys/tasks/remove_keys.yml new file mode 100644 index 0000000000..121cd88cfd --- /dev/null +++ b/tests/integration/targets/filter_remove_keys/tasks/remove_keys.yml @@ -0,0 +1,79 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Debug ansible_version + ansible.builtin.debug: + var: ansible_version + when: not quite_test | d(true) | bool + tags: ansible_version + +- name: Test remove keys equal (default) + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + vars: + rr: "{{ list1 | community.general.remove_keys(target=tt) }}" + tt: [k0_x0, k1_x1] + tags: equal_default + +- name: Test remove keys regex string + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + vars: + rr: "{{ list1 | community.general.remove_keys(target=tt, matching_parameter=mp) }}" + mp: regex + tt: '^.*[01]_x.*$' + tags: regex_string + +- name: Test remove keys targets1 + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + loop: "{{ targets1 }}" + loop_control: + label: "{{ item.mp }}: {{ item.tt }}" + vars: + rr: "{{ list1 | community.general.remove_keys(target=item.tt, matching_parameter=item.mp) }}" + tags: targets1 + +- name: Test remove keys targets2 + ansible.builtin.assert: + that: + - (rr | difference(result2) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + loop: "{{ targets2 }}" + loop_control: + label: "{{ item.mp }}: {{ item.tt }}" + vars: + rr: "{{ list2 | community.general.remove_keys(target=item.tt, matching_parameter=item.mp) }}" + tags: targets1 diff --git a/tests/integration/targets/filter_remove_keys/vars/main.yml b/tests/integration/targets/filter_remove_keys/vars/main.yml new file mode 100644 index 0000000000..a52d09a34a --- /dev/null +++ b/tests/integration/targets/filter_remove_keys/vars/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +targets1: + - {mp: equal, tt: [k0_x0, k1_x1]} + - {mp: starts_with, tt: [k0, k1]} + - {mp: ends_with, tt: [x0, x1]} + - {mp: regex, tt: ['^.*[01]_x.*$']} + - {mp: regex, tt: '^.*[01]_x.*$'} + +list1: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + +result1: + - {k2_x2: [C0], k3_x3: foo} + - {k2_x2: [C1], k3_x3: bar} + +targets2: + - {mp: equal, tt: k0_x0} + - {mp: starts_with, tt: k0} + - {mp: ends_with, tt: x0} + - {mp: regex, tt: '^.*0_x.*$'} + +list2: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + +result2: + - {k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k1_x1: B1, k2_x2: [C1], k3_x3: bar}