[PR #8446/1d615419 backport][stable-9] Feature filter replace_keys (#8507)

Feature filter replace_keys (#8446)

* Add filter replace_keys.

* Update examples and integration tests.

* Fix examples and copyright.

* Update documentation, examples and integration tests.

* Implement #8445. Add filter replace_keys

* Fix documentation formatting.

* Fix documentation.

* Fix type(target). Formatting improved.

* Instead of a dictionary, _keys_filter_target_dict returns a list

* No target testing in _keys_filter_params
* Interface changed _keys_filter_params(data, matching_parameter)
* If there are items with equal C(before) the B(first) one will be used.

* Update remove_keys. Interface changed _keys_filter_params(data, matching_parameter)

* The target can't be empty also in _keys_filter_target_dict

* Update plugins/filter/replace_keys.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/filter/replace_keys.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/filter/replace_keys.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Test attributes before and after are strings in the iteration of target.

* Update plugins/filter/replace_keys.py

Co-authored-by: Felix Fontein <felix@fontein.de>

---------

Co-authored-by: Felix Fontein <felix@fontein.de>
(cherry picked from commit 1d61541951)

Co-authored-by: Vladimir Botka <vbotka@gmail.com>

.github/BOTMETA.yml

@@ -174,6 +174,8 @@ files:
   $filters/random_mac.py: {}
   $filters/remove_keys.py:
     maintainers: vbotka
+  $filters/replace_keys.py:
+    maintainers: vbotka
   $filters/time.py:
     maintainers: resmo
   $filters/to_days.yml:

plugins/filter/keep_keys.py

@@ -110,7 +110,7 @@ def keep_keys(data, target=None, matching_parameter='equal'):
     """keep specific keys from dictionaries in a list"""
 
     # test parameters
-    _keys_filter_params(data, target, matching_parameter)
+    _keys_filter_params(data, matching_parameter)
     # test and transform target
     tt = _keys_filter_target_str(target, matching_parameter)
 

plugins/filter/remove_keys.py

@@ -110,7 +110,7 @@ def remove_keys(data, target=None, matching_parameter='equal'):
     """remove specific keys from dictionaries in a list"""
 
     # test parameters
-    _keys_filter_params(data, target, matching_parameter)
+    _keys_filter_params(data, matching_parameter)
     # test and transform target
     tt = _keys_filter_target_str(target, matching_parameter)
 

plugins/filter/replace_keys.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Vladimir Botka <vbotka@gmail.com>
+# Copyright (c) 2024 Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    name: replace_keys
+    short_description: Replace specific keys in a list of dictionaries
+    version_added: "9.1.0"
+    author:
+      - Vladimir Botka (@vbotka)
+      - Felix Fontein (@felixfontein)
+    description: This filter replaces specified keys in a provided list of dictionaries.
+    options:
+      _input:
+        description:
+          - A list of dictionaries.
+          - Top level keys must be strings.
+        type: list
+        elements: dictionary
+        required: true
+      target:
+        description:
+          - A list of dictionaries with attributes C(before) and C(after).
+          - The value of O(target[].after) replaces key matching O(target[].before).
+        type: list
+        elements: dictionary
+        required: true
+        suboptions:
+          before:
+            description:
+              - A key or key pattern to change.
+              - The interpretation of O(target[].before) depends on O(matching_parameter).
+              - For a key that matches multiple O(target[].before)s, the B(first) matching O(target[].after) will be used.
+            type: str
+          after:
+            description: A matching key change to.
+            type: str
+      matching_parameter:
+        description: Specify the matching option of target keys.
+        type: str
+        default: equal
+        choices:
+          equal: Matches keys of exactly one of the O(target[].before) items.
+          starts_with: Matches keys that start with one of the O(target[].before) items.
+          ends_with: Matches keys that end with one of the O(target[].before) items.
+          regex: Matches keys that match one of the regular expressions provided in O(target[].before).
+'''
+
+EXAMPLES = '''
+  l:
+    - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+    - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
+
+  # 1) By default, replace keys that are equal any of the attributes before.
+  t:
+    - {before: k0_x0, after: a0}
+    - {before: k1_x1, after: a1}
+  r: "{{ l | community.general.replace_keys(target=t) }}"
+
+  # 2) Replace keys that starts with any of the attributes before.
+  t:
+    - {before: k0, after: a0}
+    - {before: k1, after: a1}
+  r: "{{ l | community.general.replace_keys(target=t, matching_parameter='starts_with') }}"
+
+  # 3) Replace keys that ends with any of the attributes before.
+  t:
+    - {before: x0, after: a0}
+    - {before: x1, after: a1}
+  r: "{{ l | community.general.replace_keys(target=t, matching_parameter='ends_with') }}"
+
+  # 4) Replace keys that match any regex of the attributes before.
+  t:
+    - {before: "^.*0_x.*$", after: a0}
+    - {before: "^.*1_x.*$", after: a1}
+  r: "{{ l | community.general.replace_keys(target=t, matching_parameter='regex') }}"
+
+  # The results of above examples 1-4 are all the same.
+  r:
+    - {a0: A0, a1: B0, k2_x2: [C0], k3_x3: foo}
+    - {a0: A1, a1: B1, k2_x2: [C1], k3_x3: bar}
+
+  # 5) If more keys match the same attribute before the last one will be used.
+  t:
+    - {before: "^.*_x.*$", after: X}
+  r: "{{ l | community.general.replace_keys(target=t, matching_parameter='regex') }}"
+
+  # gives
+
+  r:
+    - X: foo
+    - X: bar
+
+  # 6) If there are items with equal attribute before the first one will be used.
+  t:
+    - {before: "^.*_x.*$", after: X}
+    - {before: "^.*_x.*$", after: Y}
+  r: "{{ l | community.general.replace_keys(target=t, matching_parameter='regex') }}"
+
+  # gives
+
+  r:
+    - X: foo
+    - X: bar
+
+  # 7) If there are more matches for a key the first one will be used.
+  l:
+    - {aaa1: A, bbb1: B, ccc1: C}
+    - {aaa2: D, bbb2: E, ccc2: F}
+  t:
+    - {before: a, after: X}
+    - {before: aa, after: Y}
+  r: "{{ l | community.general.replace_keys(target=t, matching_parameter='starts_with') }}"
+
+  # gives
+
+  r:
+    - {X: A, bbb1: B, ccc1: C}
+    - {X: D, bbb2: E, ccc2: F}
+'''
+
+RETURN = '''
+  _value:
+    description: The list of dictionaries with replaced keys.
+    type: list
+    elements: dictionary
+'''
+
+from ansible_collections.community.general.plugins.plugin_utils.keys_filter import (
+    _keys_filter_params,
+    _keys_filter_target_dict)
+
+
+def replace_keys(data, target=None, matching_parameter='equal'):
+    """replace specific keys in a list of dictionaries"""
+
+    # test parameters
+    _keys_filter_params(data, matching_parameter)
+    # test and transform target
+    tz = _keys_filter_target_dict(target, matching_parameter)
+
+    if matching_parameter == 'equal':
+        def replace_key(key):
+            for b, a in tz:
+                if key == b:
+                    return a
+            return key
+    elif matching_parameter == 'starts_with':
+        def replace_key(key):
+            for b, a in tz:
+                if key.startswith(b):
+                    return a
+            return key
+    elif matching_parameter == 'ends_with':
+        def replace_key(key):
+            for b, a in tz:
+                if key.endswith(b):
+                    return a
+            return key
+    elif matching_parameter == 'regex':
+        def replace_key(key):
+            for b, a in tz:
+                if b.match(key):
+                    return a
+            return key
+
+    return [dict((replace_key(k), v) for k, v in d.items()) for d in data]
+
+
+class FilterModule(object):
+
+    def filters(self):
+        return {
+            'replace_keys': replace_keys,
+        }

plugins/plugin_utils/keys_filter.py

@@ -13,10 +13,9 @@ from ansible.module_utils.six import string_types
 from ansible.module_utils.common._collections_compat import Mapping, Sequence
 
 
-def _keys_filter_params(data, target, matching_parameter):
+def _keys_filter_params(data, matching_parameter):
     """test parameters:
        * data must be a list of dictionaries. All keys must be strings.
-    * target must be a non-empty sequence.
        * matching_parameter is member of a list.
     """
 
@@ -37,31 +36,33 @@ def _keys_filter_params(data, target, matching_parameter):
             msg = "Top level keys must be strings. keys: %s"
             raise AnsibleFilterError(msg % elem.keys())
 
-    if not isinstance(target, Sequence):
-        msg = ("The target must be a string or a list. target is %s.")
-        raise AnsibleFilterError(msg % target)
-
-    if len(target) == 0:
-        msg = ("The target can't be empty.")
-        raise AnsibleFilterError(msg)
-
     if mp not in ml:
-        msg = ("The matching_parameter must be one of %s. matching_parameter is %s")
+        msg = "The matching_parameter must be one of %s. matching_parameter=%s"
         raise AnsibleFilterError(msg % (ml, mp))
 
     return
 
 
 def _keys_filter_target_str(target, matching_parameter):
-    """test:
-    * If target is list all items are strings
-    * If matching_parameter=regex target is a string or list with single string
-    convert and return:
+    """
+       Test:
+       * target is a non-empty string or list.
+       * If target is list all items are strings.
+       * target is a string or list with single string if matching_parameter=regex.
+       Convert target and return:
        * tuple of unique target items, or
        * tuple with single item, or
-    * compiled regex if matching_parameter=regex
+       * compiled regex if matching_parameter=regex.
     """
 
+    if not isinstance(target, Sequence):
+        msg = "The target must be a string or a list. target is %s."
+        raise AnsibleFilterError(msg % type(target))
+
+    if len(target) == 0:
+        msg = "The target can't be empty."
+        raise AnsibleFilterError(msg)
+
     if isinstance(target, list):
         for elem in target:
             if not isinstance(elem, string_types):
@@ -73,15 +74,14 @@ def _keys_filter_target_str(target, matching_parameter):
             r = target
         else:
             if len(target) > 1:
-                msg = ("Single item is required in the target list if matching_parameter is regex.")
+                msg = "Single item is required in the target list if matching_parameter=regex."
                 raise AnsibleFilterError(msg)
             else:
                 r = target[0]
         try:
             tt = re.compile(r)
         except re.error:
-            msg = ("The target must be a valid regex if matching_parameter is regex."
-                   " target is %s")
+            msg = "The target must be a valid regex if matching_parameter=regex. target is %s"
             raise AnsibleFilterError(msg % r)
     elif isinstance(target, string_types):
         tt = (target, )
@@ -92,17 +92,50 @@ def _keys_filter_target_str(target, matching_parameter):
 
 
 def _keys_filter_target_dict(target, matching_parameter):
-    """test:
-    * target is a list of dictionaries
-    * ...
+    """
+       Test:
+       * target is a list of dictionaries with attributes 'after' and 'before'.
+       * Attributes 'before' must be valid regex if matching_parameter=regex.
+       * Otherwise, the attributes 'before' must be strings.
+       Convert target and return:
+       * iterator that aggregates attributes 'before' and 'after', or
+       * iterator that aggregates compiled regex of attributes 'before' and 'after' if matching_parameter=regex.
     """
 
-    # TODO: Complete and use this in filter replace_keys
+    if not isinstance(target, list):
+        msg = "The target must be a list. target is %s."
+        raise AnsibleFilterError(msg % (target, type(target)))
+
+    if len(target) == 0:
+        msg = "The target can't be empty."
+        raise AnsibleFilterError(msg)
 
-    if isinstance(target, list):
     for elem in target:
         if not isinstance(elem, Mapping):
-                msg = "The target items must be dictionary. %s is %s"
+            msg = "The target items must be dictionaries. %s is %s"
             raise AnsibleFilterError(msg % (elem, type(elem)))
+        if not all(k in elem for k in ('before', 'after')):
+            msg = "All dictionaries in target must include attributes: after, before."
+            raise AnsibleFilterError(msg)
+        if not isinstance(elem['before'], string_types):
+            msg = "The attributes before must be strings. %s is %s"
+            raise AnsibleFilterError(msg % (elem['before'], type(elem['before'])))
+        if not isinstance(elem['after'], string_types):
+            msg = "The attributes after must be strings. %s is %s"
+            raise AnsibleFilterError(msg % (elem['after'], type(elem['after'])))
 
-    return
+    before = [d['before'] for d in target]
+    after = [d['after'] for d in target]
+
+    if matching_parameter == 'regex':
+        try:
+            tr = map(re.compile, before)
+            tz = list(zip(tr, after))
+        except re.error:
+            msg = ("The attributes before must be valid regex if matching_parameter=regex."
+                   " Not all items are valid regex in: %s")
+            raise AnsibleFilterError(msg % before)
+    else:
+        tz = list(zip(before, after))
+
+    return tz

tests/integration/targets/filter_replace_keys/aliases

@@ -0,0 +1,5 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/2

tests/integration/targets/filter_replace_keys/tasks/fn-test-replace_keys.yml

@@ -0,0 +1,21 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test replace keys
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(item.result) | length) == 0
+    success_msg: |
+      [OK]  {{ item.label }}
+      result:
+        {{ rr | to_nice_yaml(indent=2) | indent(2) }}
+    fail_msg: |
+      [ERR] {{ item.label }}
+      result:
+        {{ rr | to_nice_yaml(indent=2) | indent(2) }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  vars:
+    rr: "{{ item.data |
+            community.general.replace_keys(target=item.target, matching_parameter=item.match) }}"

tests/integration/targets/filter_replace_keys/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test replace_keys
+  import_tasks: replace_keys.yml

tests/integration/targets/filter_replace_keys/tasks/replace_keys.yml

@@ -0,0 +1,56 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Debug ansible_version
+  ansible.builtin.debug:
+    var: ansible_version
+  when: not quiet_test | d(true) | bool
+  tags: ansible_version
+
+- name: Test replace keys equal (default)
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(result1) | length) == 0
+    success_msg: |
+      [OK]  result:
+      {{ rr | to_yaml }}
+    fail_msg: |
+      [ERR] result:
+      {{ rr | to_yaml }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  vars:
+    rr: "{{ list1 | community.general.replace_keys(target=tt) }}"
+    tt:
+      - {before: k0_x0, after: a0}
+      - {before: k1_x1, after: a1}
+  tags: equal_default
+
+- name: Test replace keys targets1
+  ansible.builtin.assert:
+    that:
+      - (rr | difference(result1) | length) == 0
+    success_msg: |
+      [OK]  result:
+      {{ rr | to_yaml }}
+    fail_msg: |
+      [ERR] result:
+      {{ rr | to_yaml }}
+    quiet: "{{ quiet_test | d(true) | bool }}"
+  loop: "{{ targets1 | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  vars:
+    rr: "{{ list1 | community.general.replace_keys(target=item.value, matching_parameter=item.key) }}"
+  tags: targets1
+
+- name: Test replace keys targets2
+  include_tasks:
+    file: fn-test-replace_keys.yml
+    apply:
+      tags: targets2
+  loop: "{{ targets2 }}"
+  loop_control:
+    label: "{{ item.label }}"
+  tags: targets2

tests/integration/targets/filter_replace_keys/vars/main.yml

@@ -0,0 +1,58 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+list1:
+  - {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
+  - {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
+
+result1:
+  - {a0: A0, a1: B0, k2_x2: [C0], k3_x3: foo}
+  - {a0: A1, a1: B1, k2_x2: [C1], k3_x3: bar}
+
+targets1:
+  equal:
+    - {before: k0_x0, after: a0}
+    - {before: k1_x1, after: a1}
+  starts_with:
+    - {before: k0, after: a0}
+    - {before: k1, after: a1}
+  ends_with:
+    - {before: x0, after: a0}
+    - {before: x1, after: a1}
+  regex:
+    - {before: "^.*0_x.*$", after: a0}
+    - {before: "^.*1_x.*$", after: a1}
+
+list2:
+  - {aaa1: A, bbb1: B, ccc1: C}
+  - {aaa2: D, bbb2: E, ccc2: F}
+
+targets2:
+  - label: If more keys match the same attribute before the last one will be used.
+    match: regex
+    target:
+      - {before: "^.*_x.*$", after: X}
+    data: "{{ list1 }}"
+    result:
+      - X: foo
+      - X: bar
+  - label: If there are items with equal attribute before the first one will be used.
+    match: regex
+    target:
+      - {before: "^.*_x.*$", after: X}
+      - {before: "^.*_x.*$", after: Y}
+    data: "{{ list1 }}"
+    result:
+      - X: foo
+      - X: bar
+  - label: If there are more matches for a key the first one will be used.
+    match: starts_with
+    target:
+      - {before: a, after: X}
+      - {before: aa, after: Y}
+    data: "{{ list2 }}"
+    result:
+      - {X: A, bbb1: B, ccc1: C}
+      - {X: D, bbb2: E, ccc2: F}