From 6f8f12f762365686e18ce4a2bb0847b1c8b79cc7 Mon Sep 17 00:00:00 2001 From: Vladimir Botka Date: Tue, 4 Jun 2024 06:01:25 +0200 Subject: [PATCH] Feature filter keep_keys (#8456) * Add filter keep_keys. Implement feature request #8438 * Fix comment indentation. * Fix regex reference. * Fix indentation. * Fix isinstance list. * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein * Update plugins/filter/keep_keys.py Co-authored-by: Felix Fontein * Update documentation, examples, and integration tests. * _keys_filter_target_str returns tuple of unique target strings if target is list. Update documentation, function comments, and error messages. * Sort maintainers. * Update plugins/filter/keep_keys.py Co-authored-by: Felix Fontein * Update examples with explicit collection. --------- Co-authored-by: Felix Fontein --- .github/BOTMETA.yml | 4 + plugins/filter/keep_keys.py | 138 ++++++++++++++++++ plugins/plugin_utils/keys_filter.py | 108 ++++++++++++++ .../targets/filter_keep_keys/aliases | 5 + .../filter_keep_keys/tasks/keep_keys.yml | 79 ++++++++++ .../targets/filter_keep_keys/tasks/main.yml | 7 + .../targets/filter_keep_keys/vars/main.yml | 33 +++++ 7 files changed, 374 insertions(+) create mode 100644 plugins/filter/keep_keys.py create mode 100644 plugins/plugin_utils/keys_filter.py create mode 100644 tests/integration/targets/filter_keep_keys/aliases create mode 100644 tests/integration/targets/filter_keep_keys/tasks/keep_keys.yml create mode 100644 tests/integration/targets/filter_keep_keys/tasks/main.yml create mode 100644 tests/integration/targets/filter_keep_keys/vars/main.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 7f98718772..ef1b879de3 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -157,6 +157,8 @@ files: $filters/jc.py: maintainers: kellyjonbrazil $filters/json_query.py: {} + $filters/keep_keys.py: + maintainers: vbotka $filters/lists.py: maintainers: cfiehe $filters/lists_difference.yml: @@ -1417,6 +1419,8 @@ files: ignore: matze labels: zypper maintainers: $team_suse + $plugin_utils/keys_filter.py: + maintainers: vbotka $plugin_utils/unsafe.py: maintainers: felixfontein $tests/a_module.py: diff --git a/plugins/filter/keep_keys.py b/plugins/filter/keep_keys.py new file mode 100644 index 0000000000..009e986ab2 --- /dev/null +++ b/plugins/filter/keep_keys.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2024 Vladimir Botka +# Copyright (c) 2024 Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: keep_keys + short_description: Keep specific keys from dictionaries in a list + version_added: "9.1.0" + author: + - Vladimir Botka (@vbotka) + - Felix Fontein (@felixfontein) + description: This filter keeps only specified keys from a provided list of dictionaries. + options: + _input: + description: + - A list of dictionaries. + - Top level keys must be strings. + type: list + elements: dictionary + required: true + target: + description: + - A single key or key pattern to keep, or a list of keys or keys patterns to keep. + - If O(matching_parameter=regex) there must be exactly one pattern provided. + type: raw + required: true + matching_parameter: + description: Specify the matching option of target keys. + type: str + default: equal + choices: + equal: Matches keys of exactly one of the O(target) items. + starts_with: Matches keys that start with one of the O(target) items. + ends_with: Matches keys that end with one of the O(target) items. + regex: + - Matches keys that match the regular expresion provided in O(target). + - In this case, O(target) must be a regex string or a list with single regex string. +''' + +EXAMPLES = ''' + l: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + + # 1) By default match keys that equal any of the items in the target. + t: [k0_x0, k1_x1] + r: "{{ l | community.general.keep_keys(target=t) }}" + + # 2) Match keys that start with any of the items in the target. + t: [k0, k1] + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='starts_with') }}" + + # 3) Match keys that end with any of the items in target. + t: [x0, x1] + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='ends_with') }}" + + # 4) Match keys by the regex. + t: ['^.*[01]_x.*$'] + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='regex') }}" + + # 5) Match keys by the regex. + t: '^.*[01]_x.*$' + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='regex') }}" + + # The results of above examples 1-5 are all the same. + r: + - {k0_x0: A0, k1_x1: B0} + - {k0_x0: A1, k1_x1: B1} + + # 6) By default match keys that equal the target. + t: k0_x0 + r: "{{ l | community.general.keep_keys(target=t) }}" + + # 7) Match keys that start with the target. + t: k0 + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='starts_with') }}" + + # 8) Match keys that end with the target. + t: x0 + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='ends_with') }}" + + # 9) Match keys by the regex. + t: '^.*0_x.*$' + r: "{{ l | community.general.keep_keys(target=t, matching_parameter='regex') }}" + + # The results of above examples 6-9 are all the same. + r: + - {k0_x0: A0} + - {k0_x0: A1} +''' + +RETURN = ''' + _value: + description: The list of dictionaries with selected keys. + type: list + elements: dictionary +''' + +from ansible_collections.community.general.plugins.plugin_utils.keys_filter import ( + _keys_filter_params, + _keys_filter_target_str) + + +def keep_keys(data, target=None, matching_parameter='equal'): + """keep specific keys from dictionaries in a list""" + + # test parameters + _keys_filter_params(data, target, matching_parameter) + # test and transform target + tt = _keys_filter_target_str(target, matching_parameter) + + if matching_parameter == 'equal': + def keep_key(key): + return key in tt + elif matching_parameter == 'starts_with': + def keep_key(key): + return key.startswith(tt) + elif matching_parameter == 'ends_with': + def keep_key(key): + return key.endswith(tt) + elif matching_parameter == 'regex': + def keep_key(key): + return tt.match(key) is not None + + return [dict((k, v) for k, v in d.items() if keep_key(k)) for d in data] + + +class FilterModule(object): + + def filters(self): + return { + 'keep_keys': keep_keys, + } diff --git a/plugins/plugin_utils/keys_filter.py b/plugins/plugin_utils/keys_filter.py new file mode 100644 index 0000000000..37b7611c50 --- /dev/null +++ b/plugins/plugin_utils/keys_filter.py @@ -0,0 +1,108 @@ +# Copyright (c) 2024 Vladimir Botka +# Copyright (c) 2024 Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re + +from ansible.errors import AnsibleFilterError +from ansible.module_utils.six import string_types +from ansible.module_utils.common._collections_compat import Mapping, Sequence + + +def _keys_filter_params(data, target, matching_parameter): + """test parameters: + * data must be a list of dictionaries. All keys must be strings. + * target must be a non-empty sequence. + * matching_parameter is member of a list. + """ + + mp = matching_parameter + ml = ['equal', 'starts_with', 'ends_with', 'regex'] + + if not isinstance(data, Sequence): + msg = "First argument must be a list. %s is %s" + raise AnsibleFilterError(msg % (data, type(data))) + + for elem in data: + if not isinstance(elem, Mapping): + msg = "The data items must be dictionaries. %s is %s" + raise AnsibleFilterError(msg % (elem, type(elem))) + + for elem in data: + if not all(isinstance(item, string_types) for item in elem.keys()): + msg = "Top level keys must be strings. keys: %s" + raise AnsibleFilterError(msg % elem.keys()) + + if not isinstance(target, Sequence): + msg = ("The target must be a string or a list. target is %s.") + raise AnsibleFilterError(msg % target) + + if len(target) == 0: + msg = ("The target can't be empty.") + raise AnsibleFilterError(msg) + + if mp not in ml: + msg = ("The matching_parameter must be one of %s. matching_parameter is %s") + raise AnsibleFilterError(msg % (ml, mp)) + + return + + +def _keys_filter_target_str(target, matching_parameter): + """test: + * If target is list all items are strings + * If matching_parameter=regex target is a string or list with single string + convert and return: + * tuple of unique target items, or + * tuple with single item, or + * compiled regex if matching_parameter=regex + """ + + if isinstance(target, list): + for elem in target: + if not isinstance(elem, string_types): + msg = "The target items must be strings. %s is %s" + raise AnsibleFilterError(msg % (elem, type(elem))) + + if matching_parameter == 'regex': + if isinstance(target, string_types): + r = target + else: + if len(target) > 1: + msg = ("Single item is required in the target list if matching_parameter is regex.") + raise AnsibleFilterError(msg) + else: + r = target[0] + try: + tt = re.compile(r) + except re.error: + msg = ("The target must be a valid regex if matching_parameter is regex." + " target is %s") + raise AnsibleFilterError(msg % r) + elif isinstance(target, string_types): + tt = (target, ) + else: + tt = tuple(set(target)) + + return tt + + +def _keys_filter_target_dict(target, matching_parameter): + """test: + * target is a list of dictionaries + * ... + """ + + # TODO: Complete and use this in filter replace_keys + + if isinstance(target, list): + for elem in target: + if not isinstance(elem, Mapping): + msg = "The target items must be dictionary. %s is %s" + raise AnsibleFilterError(msg % (elem, type(elem))) + + return diff --git a/tests/integration/targets/filter_keep_keys/aliases b/tests/integration/targets/filter_keep_keys/aliases new file mode 100644 index 0000000000..12d1d6617e --- /dev/null +++ b/tests/integration/targets/filter_keep_keys/aliases @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/2 diff --git a/tests/integration/targets/filter_keep_keys/tasks/keep_keys.yml b/tests/integration/targets/filter_keep_keys/tasks/keep_keys.yml new file mode 100644 index 0000000000..94825c9d61 --- /dev/null +++ b/tests/integration/targets/filter_keep_keys/tasks/keep_keys.yml @@ -0,0 +1,79 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Debug ansible_version + ansible.builtin.debug: + var: ansible_version + when: not quite_test | d(true) | bool + tags: ansible_version + +- name: Test keep keys equal (default) + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + vars: + rr: "{{ list1 | community.general.keep_keys(target=tt) }}" + tt: [k0_x0, k1_x1] + tags: equal_default + +- name: Test keep keys regex string + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + vars: + rr: "{{ list1 | community.general.keep_keys(target=tt, matching_parameter=mp) }}" + mp: regex + tt: '^.*[01]_x.*$' + tags: regex_string + +- name: Test keep keys targets1 + ansible.builtin.assert: + that: + - (rr | difference(result1) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + loop: "{{ targets1 }}" + loop_control: + label: "{{ item.mp }}: {{ item.tt }}" + vars: + rr: "{{ list1 | community.general.keep_keys(target=item.tt, matching_parameter=item.mp) }}" + tags: targets1 + +- name: Test keep keys targets2 + ansible.builtin.assert: + that: + - (rr | difference(result2) | length) == 0 + success_msg: | + [OK] result: + {{ rr | to_yaml }} + fail_msg: | + [ERR] result: + {{ rr | to_yaml }} + quiet: "{{ quiet_test | d(true) | bool }}" + loop: "{{ targets2 }}" + loop_control: + label: "{{ item.mp }}: {{ item.tt }}" + vars: + rr: "{{ list2 | community.general.keep_keys(target=item.tt, matching_parameter=item.mp) }}" + tags: targets2 diff --git a/tests/integration/targets/filter_keep_keys/tasks/main.yml b/tests/integration/targets/filter_keep_keys/tasks/main.yml new file mode 100644 index 0000000000..23457d1e11 --- /dev/null +++ b/tests/integration/targets/filter_keep_keys/tasks/main.yml @@ -0,0 +1,7 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test keep_keys + import_tasks: keep_keys.yml diff --git a/tests/integration/targets/filter_keep_keys/vars/main.yml b/tests/integration/targets/filter_keep_keys/vars/main.yml new file mode 100644 index 0000000000..b25325253d --- /dev/null +++ b/tests/integration/targets/filter_keep_keys/vars/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +targets1: + - {mp: equal, tt: [k0_x0, k1_x1]} + - {mp: starts_with, tt: [k0, k1]} + - {mp: ends_with, tt: [x0, x1]} + - {mp: regex, tt: ['^.*[01]_x.*$']} + - {mp: regex, tt: '^.*[01]_x.*$'} + +list1: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + +result1: + - {k0_x0: A0, k1_x1: B0} + - {k0_x0: A1, k1_x1: B1} + +targets2: + - {mp: equal, tt: k0_x0} + - {mp: starts_with, tt: k0} + - {mp: ends_with, tt: x0} + - {mp: regex, tt: '^.*0_x.*$'} + +list2: + - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo} + - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar} + +result2: + - {k0_x0: A0} + - {k0_x0: A1}