[PR #8443/06f13e79 backport][stable-9] Feature filter remove_keys (#8473)

Feature filter remove_keys (#8443)

* Add filter remove_keys.

* Add filter remove_keys integration test, fragment, and maintainer.

* Update with plugins/plugin_utils/keys_filter.py

* Update according PR #8456

* Update maintainers.

* Fix typo in return doc.

* Remove local keys_filter.py. Then rebase.

* Add local keys_filter.py

* Update plugins/filter/remove_keys.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/filter/remove_keys.py

Co-authored-by: Felix Fontein <felix@fontein.de>

---------

Co-authored-by: Felix Fontein <felix@fontein.de>
(cherry picked from commit 06f13e79b1)

Co-authored-by: Vladimir Botka <vbotka@gmail.com>

.github/BOTMETA.yml

@@ -172,6 +172,8 @@ files:
   $filters/lists_union.yml:
     maintainers: cfiehe
   $filters/random_mac.py: {}
+  $filters/remove_keys.py:
+    maintainers: vbotka
   $filters/time.py:
     maintainers: resmo
   $filters/to_days.yml:

plugins/filter/remove_keys.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Vladimir Botka <vbotka@gmail.com>
+# Copyright (c) 2024 Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    name: remove_keys
+    short_description: Remove specific keys from dictionaries in a list
+    version_added: "9.1.0"
+    author:
+      - Vladimir Botka (@vbotka)
+      - Felix Fontein (@felixfontein)
+    description: This filter removes only specified keys from a provided list of dictionaries.
+    options:
+      _input:
+        description:
+          - A list of dictionaries.
+          - Top level keys must be strings.
+        type: list
+        elements: dictionary
+        required: true
+      target:
+        description:
+          - A single key or key pattern to remove, or a list of keys or keys patterns to remove.
+          - If O(matching_parameter=regex) there must be exactly one pattern provided.
+        type: raw
+        required: true
+      matching_parameter:
+        description: Specify the matching option of target keys.
+        type: str
+        default: equal
+        choices:
+          equal: Matches keys of exactly one of the O(target) items.
+          starts_with: Matches keys that start with one of the O(target) items.
+          ends_with: Matches keys that end with one of the O(target) items.
+          regex:
+            - Matches keys that match the regular expresion provided in O(target).
+            - In this case, O(target) must be a regex string or a list with single regex string.
+'''
+
+EXAMPLES = '''
+  l:
+    - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+    - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
+
+  # 1) By default match keys that equal any of the items in the target.
+  t: [k0_x0, k1_x1]
+  r: "{{ l | community.general.remove_keys(target=t) }}"
+
+  # 2) Match keys that start with any of the items in the target.
+  t: [k0, k1]
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='starts_with') }}"
+
+  # 3) Match keys that end with any of the items in target.
+  t: [x0, x1]
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='ends_with') }}"
+
+  # 4) Match keys by the regex.
+  t: ['^.*[01]_x.*$']
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='regex') }}"
+
+  # 5) Match keys by the regex.
+  t: '^.*[01]_x.*$'
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='regex') }}"
+
+  # The results of above examples 1-5 are all the same.
+  r:
+    - {k2_x2: [C0], k3_x3: foo}
+    - {k2_x2: [C1], k3_x3: bar}
+
+  # 6) By default match keys that equal the target.
+  t: k0_x0
+  r: "{{ l | community.general.remove_keys(target=t) }}"
+
+  # 7) Match keys that start with the target.
+  t: k0
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='starts_with') }}"
+
+  # 8) Match keys that end with the target.
+  t: x0
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='ends_with') }}"
+
+  # 9) Match keys by the regex.
+  t: '^.*0_x.*$'
+  r: "{{ l | community.general.remove_keys(target=t, matching_parameter='regex') }}"
+
+  # The results of above examples 6-9 are all the same.
+  r:
+    - {k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+    - {k1_x1: B1, k2_x2: [C1], k3_x3: bar}
+'''
+
+RETURN = '''
+  _value:
+    description: The list of dictionaries with selected keys removed.
+    type: list
+    elements: dictionary
+'''
+
+from ansible_collections.community.general.plugins.plugin_utils.keys_filter import (
+    _keys_filter_params,
+    _keys_filter_target_str)
+
+
+def remove_keys(data, target=None, matching_parameter='equal'):
+    """remove specific keys from dictionaries in a list"""
+
+    # test parameters
+    _keys_filter_params(data, target, matching_parameter)
+    # test and transform target
+    tt = _keys_filter_target_str(target, matching_parameter)
+
+    if matching_parameter == 'equal':
+        def keep_key(key):
+            return key not in tt
+    elif matching_parameter == 'starts_with':
+        def keep_key(key):
+            return not key.startswith(tt)
+    elif matching_parameter == 'ends_with':
+        def keep_key(key):
+            return not key.endswith(tt)
+    elif matching_parameter == 'regex':
+        def keep_key(key):
+            return tt.match(key) is None
+
+    return [dict((k, v) for k, v in d.items() if keep_key(k)) for d in data]
+
+
+class FilterModule(object):
+
+    def filters(self):
+        return {
+            'remove_keys': remove_keys,
+        }

tests/integration/targets/filter_remove_keys/aliases

@@ -0,0 +1,5 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/2

tests/integration/targets/filter_remove_keys/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test remove_keys
+  import_tasks: remove_keys.yml

tests/integration/targets/filter_remove_keys/tasks/remove_keys.yml

@@ -0,0 +1,79 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Debug ansible_version
+  ansible.builtin.debug:
+    var: ansible_version
+  when: not quite_test | d(true) | bool
+  tags: ansible_version
+
+- name: Test remove keys equal (default)
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(result1) | length) == 0
+    success_msg: |
+      [OK]  result:
+      {{ rr | to_yaml }}
+    fail_msg: |
+      [ERR] result:
+      {{ rr | to_yaml }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  vars:
+    rr: "{{ list1 | community.general.remove_keys(target=tt) }}"
+    tt: [k0_x0, k1_x1]
+  tags: equal_default
+
+- name: Test remove keys regex string
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(result1) | length) == 0
+    success_msg: |
+      [OK]  result:
+      {{ rr | to_yaml }}
+    fail_msg: |
+      [ERR] result:
+      {{ rr | to_yaml }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  vars:
+    rr: "{{ list1 | community.general.remove_keys(target=tt, matching_parameter=mp) }}"
+    mp: regex
+    tt: '^.*[01]_x.*$'
+  tags: regex_string
+
+- name: Test remove keys targets1
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(result1) | length) == 0
+    success_msg: |
+      [OK]  result:
+      {{ rr | to_yaml }}
+    fail_msg: |
+      [ERR] result:
+      {{ rr | to_yaml }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  loop: "{{ targets1 }}"
+  loop_control:
+    label: "{{ item.mp }}: {{ item.tt }}"
+  vars:
+    rr: "{{ list1 | community.general.remove_keys(target=item.tt, matching_parameter=item.mp) }}"
+  tags: targets1
+
+- name: Test remove keys targets2
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(result2) | length) == 0
+    success_msg: |
+      [OK]  result:
+      {{ rr | to_yaml }}
+    fail_msg: |
+      [ERR] result:
+      {{ rr | to_yaml }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  loop: "{{ targets2 }}"
+  loop_control:
+    label: "{{ item.mp }}: {{ item.tt }}"
+  vars:
+    rr: "{{ list2 | community.general.remove_keys(target=item.tt, matching_parameter=item.mp) }}"
+  tags: targets1

tests/integration/targets/filter_remove_keys/vars/main.yml

@@ -0,0 +1,33 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+targets1:
+  - {mp: equal, tt: [k0_x0, k1_x1]}
+  - {mp: starts_with, tt: [k0, k1]}
+  - {mp: ends_with, tt: [x0, x1]}
+  - {mp: regex, tt: ['^.*[01]_x.*$']}
+  - {mp: regex, tt: '^.*[01]_x.*$'}
+
+list1:
+  - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+  - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
+
+result1:
+  - {k2_x2: [C0], k3_x3: foo}
+  - {k2_x2: [C1], k3_x3: bar}
+
+targets2:
+  - {mp: equal, tt: k0_x0}
+  - {mp: starts_with, tt: k0}
+  - {mp: ends_with, tt: x0}
+  - {mp: regex, tt: '^.*0_x.*$'}
+
+list2:
+  - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+  - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
+
+result2:
+  - {k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+  - {k1_x1: B1, k2_x2: [C1], k3_x3: bar}