diff --git a/changelogs/fragments/4058-lists_mergeby-add-parameters.yml b/changelogs/fragments/4058-lists_mergeby-add-parameters.yml new file mode 100644 index 0000000000..3f73b8f3ab --- /dev/null +++ b/changelogs/fragments/4058-lists_mergeby-add-parameters.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - lists_mergeby filter plugin - add parameters ``list_merge`` and ``recursive``. These are only supported when used with ansible-base 2.10 or ansible-core, but not with Ansible 2.9 (https://github.com/ansible-collections/community.general/pull/4058). diff --git a/docs/docsite/helper/lists_mergeby/examples.rst.j2 b/docs/docsite/helper/lists_mergeby/examples.rst.j2 new file mode 100644 index 0000000000..d6abeb1108 --- /dev/null +++ b/docs/docsite/helper/lists_mergeby/examples.rst.j2 @@ -0,0 +1,8 @@ +{% for i in examples %} +{{ i.label }} + +.. code-block:: {{ i.lang }} + + {{ lookup('file', source_path ~ i.file)|indent(2) }} + +{% endfor %} diff --git a/docs/docsite/helper/lists_mergeby/examples.yml b/docs/docsite/helper/lists_mergeby/examples.yml new file mode 100644 index 0000000000..3a8110a098 --- /dev/null +++ b/docs/docsite/helper/lists_mergeby/examples.yml @@ -0,0 +1,38 @@ +--- +examples: + - label: 'Example ``list_merge=replace`` (default):' + file: example-003.yml + lang: 'yaml+jinja' + - label: 'This produces:' + file: example-003.out + lang: 'yaml' + - label: 'Example ``list_merge=keep``:' + file: example-004.yml + lang: 'yaml+jinja' + - label: 'This produces:' + file: example-004.out + lang: 'yaml' + - label: 'Example ``list_merge=append``:' + file: example-005.yml + lang: 'yaml+jinja' + - label: 'This produces:' + file: example-005.out + lang: 'yaml' + - label: 'Example ``list_merge=prepend``:' + file: example-006.yml + lang: 'yaml+jinja' + - label: 'This produces:' + file: example-006.out + lang: 'yaml' + - label: 'Example ``list_merge=append_rp``:' + file: example-007.yml + lang: 'yaml+jinja' + - label: 'This produces:' + file: example-007.out + lang: 'yaml' + - label: 'Example ``list_merge=prepend_rp``:' + file: example-008.yml + lang: 'yaml+jinja' + - label: 'This produces:' + file: example-008.out + lang: 'yaml' diff --git a/docs/docsite/helper/lists_mergeby/playbook.yml b/docs/docsite/helper/lists_mergeby/playbook.yml new file mode 100644 index 0000000000..c31e1853ff --- /dev/null +++ b/docs/docsite/helper/lists_mergeby/playbook.yml @@ -0,0 +1,41 @@ +--- +# The following runs all examples: +# +# ANSIBLE_STDOUT_CALLBACK=community.general.yaml ansible-playbook playbook.yml -e examples=true +# +# You need to copy the YAML output of example-XXX.yml into example-XXX.out. +# +# The following generates examples.rst out of the .out files: +# +# ansible-playbook playbook.yml -e template=true +- hosts: localhost + gather_facts: false + vars: + source_path: ../../rst/examples/lists_mergeby/ + tasks: + + - block: + - import_tasks: '{{ source_path }}example-001.yml' + tags: t001 + - import_tasks: '{{ source_path }}example-002.yml' + tags: t002 + - import_tasks: '{{ source_path }}example-003.yml' + tags: t003 + - import_tasks: '{{ source_path }}example-004.yml' + tags: t004 + - import_tasks: '{{ source_path }}example-005.yml' + tags: t005 + - import_tasks: '{{ source_path }}example-006.yml' + tags: t006 + - import_tasks: '{{ source_path }}example-007.yml' + tags: t007 + - import_tasks: '{{ source_path }}example-008.yml' + tags: t008 + when: examples|d(false)|bool + + - block: + - include_vars: examples.yml + - template: + src: examples.rst.j2 + dest: examples.rst + when: template|d(false)|bool diff --git a/docs/docsite/rst/examples/lists_mergeby/example-001.out b/docs/docsite/rst/examples/lists_mergeby/example-001.out new file mode 100644 index 0000000000..879c2cadb7 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-001.out @@ -0,0 +1,10 @@ + list3: + - extra: false + name: bar + - name: baz + path: /baz + - extra: true + name: foo + path: /foo + - extra: true + name: meh diff --git a/docs/docsite/rst/examples/lists_mergeby/example-001.yml b/docs/docsite/rst/examples/lists_mergeby/example-001.yml new file mode 100644 index 0000000000..3284dbddfd --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-001.yml @@ -0,0 +1,20 @@ +--- +- name: Merge two lists by common attribute 'name' + set_fact: + list3: "{{ list1| + community.general.lists_mergeby(list2, 'name') }}" + vars: + list1: + - name: foo + extra: true + - name: bar + extra: false + - name: meh + extra: true + list2: + - name: foo + path: /foo + - name: baz + path: /baz +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-002.out b/docs/docsite/rst/examples/lists_mergeby/example-002.out new file mode 100644 index 0000000000..879c2cadb7 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-002.out @@ -0,0 +1,10 @@ + list3: + - extra: false + name: bar + - name: baz + path: /baz + - extra: true + name: foo + path: /foo + - extra: true + name: meh diff --git a/docs/docsite/rst/examples/lists_mergeby/example-002.yml b/docs/docsite/rst/examples/lists_mergeby/example-002.yml new file mode 100644 index 0000000000..d8a3ddf529 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-002.yml @@ -0,0 +1,20 @@ +--- +- name: Merge two lists by common attribute 'name' + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name') }}" + vars: + list1: + - name: foo + extra: true + - name: bar + extra: false + - name: meh + extra: true + list2: + - name: foo + path: /foo + - name: baz + path: /baz +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-003.out b/docs/docsite/rst/examples/lists_mergeby/example-003.out new file mode 100644 index 0000000000..62ba8def2c --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-003.out @@ -0,0 +1,14 @@ + list3: + - name: myname01 + param01: + list: + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 3 + - 4 + - 4 + - key: value diff --git a/docs/docsite/rst/examples/lists_mergeby/example-003.yml b/docs/docsite/rst/examples/lists_mergeby/example-003.yml new file mode 100644 index 0000000000..7dbbcae63e --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-003.yml @@ -0,0 +1,28 @@ +--- +- name: Merge recursive by 'name', replace lists (default) + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true) }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-004.out b/docs/docsite/rst/examples/lists_mergeby/example-004.out new file mode 100644 index 0000000000..18d1d12a02 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-004.out @@ -0,0 +1,14 @@ + list3: + - name: myname01 + param01: + list: + - default_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-004.yml b/docs/docsite/rst/examples/lists_mergeby/example-004.yml new file mode 100644 index 0000000000..26c1089137 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-004.yml @@ -0,0 +1,29 @@ +--- +- name: Merge recursive by 'name', keep lists + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='keep') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-005.out b/docs/docsite/rst/examples/lists_mergeby/example-005.out new file mode 100644 index 0000000000..5fff8638db --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-005.out @@ -0,0 +1,19 @@ + list3: + - name: myname01 + param01: + list: + - default_value + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + - 3 + - 4 + - 4 + - key: value diff --git a/docs/docsite/rst/examples/lists_mergeby/example-005.yml b/docs/docsite/rst/examples/lists_mergeby/example-005.yml new file mode 100644 index 0000000000..f2d76059e8 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-005.yml @@ -0,0 +1,29 @@ +--- +- name: Merge recursive by 'name', append lists + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='append') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-006.out b/docs/docsite/rst/examples/lists_mergeby/example-006.out new file mode 100644 index 0000000000..2c0e0652f1 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-006.out @@ -0,0 +1,19 @@ + list3: + - name: myname01 + param01: + list: + - patch_value + - default_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 3 + - 4 + - 4 + - key: value + - 1 + - 1 + - 2 + - 3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-006.yml b/docs/docsite/rst/examples/lists_mergeby/example-006.yml new file mode 100644 index 0000000000..91e18a00a1 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-006.yml @@ -0,0 +1,29 @@ +--- +- name: Merge recursive by 'name', prepend lists + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='prepend') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-007.out b/docs/docsite/rst/examples/lists_mergeby/example-007.out new file mode 100644 index 0000000000..516ea026f8 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-007.out @@ -0,0 +1,18 @@ + list3: + - name: myname01 + param01: + list: + - default_value + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + - 4 + - 4 + - key: value diff --git a/docs/docsite/rst/examples/lists_mergeby/example-007.yml b/docs/docsite/rst/examples/lists_mergeby/example-007.yml new file mode 100644 index 0000000000..46139e58f8 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-007.yml @@ -0,0 +1,29 @@ +--- +- name: Merge recursive by 'name', append lists 'remove present' + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='append_rp') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] +- debug: + var: list3 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-008.out b/docs/docsite/rst/examples/lists_mergeby/example-008.out new file mode 100644 index 0000000000..838cee02a3 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-008.out @@ -0,0 +1,18 @@ + list3: + - name: myname01 + param01: + list: + - patch_value + - default_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 3 + - 4 + - 4 + - key: value + - 1 + - 1 + - 2 diff --git a/docs/docsite/rst/examples/lists_mergeby/example-008.yml b/docs/docsite/rst/examples/lists_mergeby/example-008.yml new file mode 100644 index 0000000000..adcdcc78a3 --- /dev/null +++ b/docs/docsite/rst/examples/lists_mergeby/example-008.yml @@ -0,0 +1,29 @@ +--- +- name: Merge recursive by 'name', prepend lists 'remove present' + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='prepend_rp') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] +- debug: + var: list3 diff --git a/docs/docsite/rst/filter_guide.rst b/docs/docsite/rst/filter_guide.rst index 18e4158d44..4ff9a3f7c6 100644 --- a/docs/docsite/rst/filter_guide.rst +++ b/docs/docsite/rst/filter_guide.rst @@ -1,3 +1,4 @@ + .. _ansible_collections.community.general.docsite.filter_guide: community.general Filter Guide @@ -247,55 +248,437 @@ This produces: Merging lists of dictionaries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you have two lists of dictionaries and want to combine them into a list of merged dictionaries, where two dictionaries are merged if they coincide in one attribute, you can use the ``lists_mergeby`` filter. +If you have two or more lists of dictionaries and want to combine them into a list of merged dictionaries, where the dictionaries are merged by an attribute, you can use the ``lists_mergeby`` filter. + +.. note:: The output of the examples in this section use the YAML callback plugin. Quoting: "Ansible output that can be quite a bit easier to read than the default JSON formatting." See :ref:`the documentation for the community.general.yaml callback plugin `. + +In the example below the lists are merged by the attribute ``name``: .. code-block:: yaml+jinja - - name: Merge two lists by common attribute 'name' - debug: - var: list1 | community.general.lists_mergeby(list2, 'name') - vars: - list1: - - name: foo - extra: true - - name: bar - extra: false - - name: meh - extra: true - list2: - - name: foo - path: /foo - - name: baz - path: /bazzz + --- + - name: Merge two lists by common attribute 'name' + set_fact: + list3: "{{ list1| + community.general.lists_mergeby(list2, 'name') }}" + vars: + list1: + - name: foo + extra: true + - name: bar + extra: false + - name: meh + extra: true + list2: + - name: foo + path: /foo + - name: baz + path: /baz + - debug: + var: list3 This produces: -.. code-block:: ansible-output +.. code-block:: yaml - TASK [Merge two lists by common attribute 'name'] **************************************** - ok: [localhost] => { - "list1 | community.general.lists_mergeby(list2, 'name')": [ - { - "extra": false, - "name": "bar" - }, - { - "name": "baz", - "path": "/bazzz" - }, - { - "extra": true, - "name": "foo", - "path": "/foo" - }, - { - "extra": true, - "name": "meh" - } - ] - } + list3: + - extra: false + name: bar + - name: baz + path: /baz + - extra: true + name: foo + path: /foo + - extra: true + name: meh + +.. versionadded:: 2.0.0 + +It is possible to use a list of lists as an input of the filter: + +.. code-block:: yaml+jinja + + --- + - name: Merge two lists by common attribute 'name' + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name') }}" + vars: + list1: + - name: foo + extra: true + - name: bar + extra: false + - name: meh + extra: true + list2: + - name: foo + path: /foo + - name: baz + path: /baz + - debug: + var: list3 + +This produces the same result as in the previous example: + +.. code-block:: yaml + + list3: + - extra: false + name: bar + - name: baz + path: /baz + - extra: true + name: foo + path: /foo + - extra: true + name: meh + +The filter also accepts two optional parameters: ``recursive`` and ``list_merge``. These parameters are only supported when used with ansible-base 2.10 or ansible-core, but not with Ansible 2.9. This is available since community.general 4.4.0. + +**recursive** + Is a boolean, default to ``False``. Should the ``community.general.lists_mergeby`` recursively merge nested hashes. Note: It does not depend on the value of the ``hash_behaviour`` setting in ``ansible.cfg``. + +**list_merge** + Is a string, its possible values are ``replace`` (default), ``keep``, ``append``, ``prepend``, ``append_rp`` or ``prepend_rp``. It modifies the behaviour of ``community.general.lists_mergeby`` when the hashes to merge contain arrays/lists. + +The examples below set ``recursive=true`` and display the differences among all six options of ``list_merge``. Functionality of the parameters is exactly the same as in the filter ``combine``. See :ref:`Combining hashes/dictionaries ` to learn details about these options. + +Example ``list_merge=replace`` (default): + +.. code-block:: yaml+jinja + + --- + - name: Merge recursive by 'name', replace lists (default) + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true) }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + - debug: + var: list3 + +This produces: + +.. code-block:: yaml + + list3: + - name: myname01 + param01: + list: + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 3 + - 4 + - 4 + - key: value + +Example ``list_merge=keep``: + +.. code-block:: yaml+jinja + + --- + - name: Merge recursive by 'name', keep lists + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='keep') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + - debug: + var: list3 + +This produces: + +.. code-block:: yaml + + list3: + - name: myname01 + param01: + list: + - default_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + +Example ``list_merge=append``: + +.. code-block:: yaml+jinja + + --- + - name: Merge recursive by 'name', append lists + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='append') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + - debug: + var: list3 + +This produces: + +.. code-block:: yaml + + list3: + - name: myname01 + param01: + list: + - default_value + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + - 3 + - 4 + - 4 + - key: value + +Example ``list_merge=prepend``: + +.. code-block:: yaml+jinja + + --- + - name: Merge recursive by 'name', prepend lists + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='prepend') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + - debug: + var: list3 + +This produces: + +.. code-block:: yaml + + list3: + - name: myname01 + param01: + list: + - patch_value + - default_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 3 + - 4 + - 4 + - key: value + - 1 + - 1 + - 2 + - 3 + +Example ``list_merge=append_rp``: + +.. code-block:: yaml+jinja + + --- + - name: Merge recursive by 'name', append lists 'remove present' + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='append_rp') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + - debug: + var: list3 + +This produces: + +.. code-block:: yaml + + list3: + - name: myname01 + param01: + list: + - default_value + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + - 4 + - 4 + - key: value + +Example ``list_merge=prepend_rp``: + +.. code-block:: yaml+jinja + + --- + - name: Merge recursive by 'name', prepend lists 'remove present' + set_fact: + list3: "{{ [list1, list2]| + community.general.lists_mergeby('name', + recursive=true, + list_merge='prepend_rp') }}" + vars: + list1: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + + list2: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + - debug: + var: list3 + +This produces: + +.. code-block:: yaml + + list3: + - name: myname01 + param01: + list: + - patch_value + - default_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 3 + - 4 + - 4 + - key: value + - 1 + - 1 + - 2 -.. versionadded: 2.0.0 Counting elements in a sequence ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/plugins/filter/list.py b/plugins/filter/list.py index 460e45194f..005e4b7c70 100644 --- a/plugins/filter/list.py +++ b/plugins/filter/list.py @@ -1,43 +1,113 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020, Vladimir Botka +# Copyright (c) 2020-2022, Vladimir Botka # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.errors import AnsibleError, AnsibleFilterError +from ansible.errors import AnsibleFilterError from ansible.module_utils.six import string_types from ansible.module_utils.common._collections_compat import Mapping, Sequence +from ansible.utils.vars import merge_hash +from ansible.release import __version__ as ansible_version +from ansible_collections.community.general.plugins.module_utils.version import LooseVersion + from collections import defaultdict from operator import itemgetter -def lists_mergeby(l1, l2, index): - ''' merge lists by attribute index. Example: - - debug: msg="{{ l1|community.general.lists_mergeby(l2, 'index')|list }}" ''' +def merge_hash_wrapper(x, y, recursive=False, list_merge='replace'): + ''' Wrapper of the function merge_hash from ansible.utils.vars. Only 2 paramaters are allowed + for Ansible 2.9 and lower.''' - if not isinstance(l1, Sequence): - raise AnsibleFilterError('First argument for community.general.lists_mergeby must be list. %s is %s' % - (l1, type(l1))) + if LooseVersion(ansible_version) < LooseVersion('2.10'): + if list_merge != 'replace' or recursive: + msg = ("Non default options of list_merge(default=replace) or recursive(default=False) " + "are not allowed in Ansible version 2.9 or lower. Ansible version is %s, " + "recursive=%s, and list_merge=%s.") + raise AnsibleFilterError(msg % (ansible_version, recursive, list_merge)) + else: + return merge_hash(x, y) + else: + return merge_hash(x, y, recursive, list_merge) - if not isinstance(l2, Sequence): - raise AnsibleFilterError('Second argument for community.general.lists_mergeby must be list. %s is %s' % - (l2, type(l2))) - if not isinstance(index, string_types): - raise AnsibleFilterError('Third argument for community.general.lists_mergeby must be string. %s is %s' % - (index, type(index))) +def list_mergeby(x, y, index, recursive=False, list_merge='replace'): + ''' Merge 2 lists by attribute 'index'. The function merge_hash from ansible.utils.vars is used. + This function is used by the function lists_mergeby. + ''' d = defaultdict(dict) - for l in (l1, l2): + for l in (x, y): for elem in l: if not isinstance(elem, Mapping): - raise AnsibleFilterError('Elements of list arguments for lists_mergeby must be dictionaries. Found {0!r}.'.format(elem)) + msg = "Elements of list arguments for lists_mergeby must be dictionaries. %s is %s" + raise AnsibleFilterError(msg % (elem, type(elem))) if index in elem.keys(): - d[elem[index]].update(elem) + d[elem[index]].update(merge_hash_wrapper(d[elem[index]], elem, recursive, list_merge)) return sorted(d.values(), key=itemgetter(index)) +def lists_mergeby(*terms, **kwargs): + ''' Merge 2 or more lists by attribute 'index'. Optional parameters 'recursive' and 'list_merge' + control the merging of the lists in values. The function merge_hash from ansible.utils.vars + is used. To learn details on how to use the parameters 'recursive' and 'list_merge' see + Ansible User's Guide chapter "Using filters to manipulate data" section "Combining + hashes/dictionaries". + + Example: + - debug: + msg: "{{ list1| + community.general.lists_mergeby(list2, + 'index', + recursive=True, + list_merge='append')| + list }}" + ''' + + recursive = kwargs.pop('recursive', False) + list_merge = kwargs.pop('list_merge', 'replace') + if kwargs: + raise AnsibleFilterError("'recursive' and 'list_merge' are the only valid keyword arguments.") + if len(terms) < 2: + raise AnsibleFilterError("At least one list and index are needed.") + + # allow the user to do `[list1, list2, ...] | lists_mergeby('index')` + flat_list = [] + for sublist in terms[:-1]: + if not isinstance(sublist, Sequence): + msg = ("All arguments before the argument index for community.general.lists_mergeby " + "must be lists. %s is %s") + raise AnsibleFilterError(msg % (sublist, type(sublist))) + if len(sublist) > 0: + if all(isinstance(l, Sequence) for l in sublist): + for item in sublist: + flat_list.append(item) + else: + flat_list.append(sublist) + lists = flat_list + + if not lists: + return [] + + if len(lists) == 1: + return lists[0] + + index = terms[-1] + + if not isinstance(index, string_types): + msg = ("First argument after the lists for community.general.lists_mergeby must be string. " + "%s is %s") + raise AnsibleFilterError(msg % (index, type(index))) + + high_to_low_prio_list_iterator = reversed(lists) + result = next(high_to_low_prio_list_iterator) + for list in high_to_low_prio_list_iterator: + result = list_mergeby(list, result, index, recursive, list_merge) + + return result + + class FilterModule(object): ''' Ansible list filters ''' diff --git a/tests/integration/targets/filter_list/tasks/lists_mergeby.yml b/tests/integration/targets/filter_list/tasks/lists_mergeby.yml deleted file mode 100644 index 903ae78c3c..0000000000 --- a/tests/integration/targets/filter_list/tasks/lists_mergeby.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- name: Test lists merged by attribute name - assert: - that: - - "(list1 | community.general.lists_mergeby(list2, 'name') | list | - difference(list3) | length) == 0" - -- name: Test list1 empty - assert: - that: - - "([] | community.general.lists_mergeby(list2, 'name') | list | - difference(list2) | length) == 0" - -- name: Test list2 empty - assert: - that: - - "(list1 | community.general.lists_mergeby([], 'name') | list | - difference(list1) | length) == 0" - -- name: Test all lists empty - assert: - that: - - "([] | community.general.lists_mergeby([], 'name') | list | - difference(list1) | length) == 0" - -- name: First argument must be list - set_fact: - my_list: "{{ {'x': 'y'} | community.general.lists_mergeby(list2, 'name') }}" - register: result - ignore_errors: true -- assert: - that: - - result is failed - - '"First argument for community.general.lists_mergeby must be list." in result.msg' - -- name: Second argument must be list - set_fact: - my_list: "{{ list1 | community.general.lists_mergeby({'x': 'y'}, 'name') }}" - register: result - ignore_errors: true -- assert: - that: - - result is failed - - '"Second argument for community.general.lists_mergeby must be list." in result.msg' - -- name: Third argument must be string - set_fact: - my_list: "{{ list1 | community.general.lists_mergeby(list2, {'x': 'y'}) }}" - register: result - ignore_errors: true -- assert: - that: - - result is failed - - '"Third argument for community.general.lists_mergeby must be string." in result.msg' - -- name: Elements of list must be dictionaries - set_fact: - my_list: "{{ list4 | community.general.lists_mergeby(list2, 'name') }}" - register: result - ignore_errors: true -- assert: - that: - - result is failed - - '"Elements of list arguments for lists_mergeby must be dictionaries." in result.msg' diff --git a/tests/integration/targets/filter_list/tasks/lists_mergeby_2-10.yml b/tests/integration/targets/filter_list/tasks/lists_mergeby_2-10.yml new file mode 100644 index 0000000000..3a53f4033b --- /dev/null +++ b/tests/integration/targets/filter_list/tasks/lists_mergeby_2-10.yml @@ -0,0 +1,140 @@ +--- + +- name: 101.Merge 2 lists by attribute name. list_merge='keep' + block: + - name: Merge 2 lists by attribute name. list_merge='keep'. set + set_fact: + my_list: "{{ [list100, list101]| + community.general.lists_mergeby('name', list_merge='keep') }}" + - name: Merge 2 lists by attribute name. list_merge='keep'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result101): + {{ my_list|difference(result101)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge 2 lists by attribute name. list_merge='keep'. assert + assert: + that: my_list | difference(result101) | length == 0 + tags: t101 + +- name: 102.Merge 2 lists by attribute name. list_merge='append' + block: + - name: Merge 2 lists by attribute name. list_merge='append'. set + set_fact: + my_list: "{{ [list100, list101]| + community.general.lists_mergeby('name', list_merge='append') }}" + - name: Merge 2 lists by attribute name. list_merge='append'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result102): + {{ my_list|difference(result102)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge 2 lists by attribute name. list_merge='append'. assert + assert: + that: my_list | difference(result102) | length == 0 + tags: t102 + +- name: 103.Merge 2 lists by attribute name. list_merge='prepend' + block: + - name: Merge 2 lists by attribute name. list_merge='prepend'. set + set_fact: + my_list: "{{ [list100, list101]| + community.general.lists_mergeby('name', list_merge='prepend') }}" + - name: Merge 2 lists by attribute name. list_merge='prepend'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result103): + {{ my_list|difference(result103)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge 2 lists by attribute name. list_merge='prepend'. assert + assert: + that: my_list | difference(result103) | length == 0 + tags: t103 + +- name: 104.Merge 2 lists by attribute name. list_merge='append_rp' + block: + - name: Merge 2 lists by attribute name. list_merge='append_rp'. set + set_fact: + my_list: "{{ [list102, list103]| + community.general.lists_mergeby('name', list_merge='append_rp') }}" + - name: Merge 2 lists by attribute name. list_merge='append_rp'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result104): + {{ my_list|difference(result104)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge 2 lists by attribute name. list_merge='append_rp'. assert + assert: + that: my_list | difference(result104) | length == 0 + tags: t104 + +- name: 105.Merge 2 lists by attribute name. list_merge='prepend_rp' + block: + - name: Merge 2 lists by attribute name. list_merge='prepend_rp'. set + set_fact: + my_list: "{{ [list102, list103]| + community.general.lists_mergeby('name', list_merge='prepend_rp') }}" + - name: Merge 2 lists by attribute name. list_merge='prepend_rp'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result105): + {{ my_list|difference(result105)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge 2 lists by attribute name. list_merge='prepend_rp'. assert + assert: + that: my_list | difference(result105) | length == 0 + tags: t105 + +# Test recursive + +- name: 200.Merge by name. recursive=True list_merge='append_rp' + block: + - name: Merge by name. recursive=True list_merge='append_rp'. set + set_fact: + my_list: "{{ [list200, list201]| + community.general.lists_mergeby('name', + recursive=True, + list_merge='append_rp') }}" + - name: Merge by name. recursive=True list_merge='append_rp'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result200): + {{ my_list|difference(result200)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge by name. recursive=True list_merge='append_rp'. assert + assert: + that: my_list | difference(result200) | length == 0 + tags: t200 + +- name: 201.Merge by name. recursive=False list_merge='append_rp' + block: + - name: Merge by name. recursive=False list_merge='append_rp'. set + set_fact: + my_list: "{{ [list200, list201]| + community.general.lists_mergeby('name', + recursive=False, + list_merge='append_rp') }}" + - name: Merge by name. recursive=False list_merge='append_rp'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result201): + {{ my_list|difference(result201)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge by name. recursive=False list_merge='append_rp'. assert + assert: + that: my_list | difference(result201) | length == 0 + tags: t201 diff --git a/tests/integration/targets/filter_list/tasks/lists_mergeby_default.yml b/tests/integration/targets/filter_list/tasks/lists_mergeby_default.yml new file mode 100644 index 0000000000..5c634a9235 --- /dev/null +++ b/tests/integration/targets/filter_list/tasks/lists_mergeby_default.yml @@ -0,0 +1,166 @@ +--- + +- name: Debug ansible_version + debug: + var: ansible_version + when: debug_test|d(false)|bool + tags: t0 + +- name: 1. Test lists merged by attribute name + block: + - name: Test lists merged by attribute name debug + debug: + msg: "{{ list1 | community.general.lists_mergeby(list2, 'name') }}" + when: debug_test|d(false)|bool + - name: Test lists merged by attribute name assert + assert: + that: + - "(list1 | community.general.lists_mergeby(list2, 'name') | list | + difference(list3) | length) == 0" + tags: t1 + +- name: 2.Test list1 empty + block: + - name: Test list1 empty debug + debug: + msg: "{{ [] | community.general.lists_mergeby(list2, 'name') }}" + when: debug_test|d(false)|bool + - name: Test list1 empty assert + assert: + that: + - "([] | community.general.lists_mergeby(list2, 'name') | list | + difference(list2) | length) == 0" + tags: t2 + +- name: 3.Test all lists empty + block: + - name: Test all lists empty debug + debug: + msg: "{{ [] | community.general.lists_mergeby([], 'name') }}" + when: debug_test|d(false)|bool + - name: Test all lists empty assert + assert: + that: + - "([] | community.general.lists_mergeby([], 'name') | list | + length) == 0" + tags: t3 + +- name: 4.First argument must be list + block: + - name: First argument must be list set + set_fact: + my_list: "{{ {'x': 'y'} | community.general.lists_mergeby(list2, 'name') }}" + register: result + ignore_errors: true + - name: First argument must be list debug + debug: + var: my_list + when: debug_test|d(false)|bool + - name: First argument must be list assert + assert: + that: + - result is failed + - '"All arguments before the argument index for community.general.lists_mergeby must be lists." in result.msg' + tags: t4 + +- name: 5.Second argument must be list + block: + - name: Second argument must be list set + set_fact: + my_list: "{{ list1 | community.general.lists_mergeby({'x': 'y'}, 'name') }}" + register: result + ignore_errors: true + - name: Second argument must be list set debug + debug: + var: my_list + when: debug_test|d(false)|bool + - name: Second argument must be list set assert + assert: + that: + - result is failed + - '"All arguments before the argument index for community.general.lists_mergeby must be lists." in result.msg' + tags: t5 + +- name: 6.First arguments after the lists must be string + block: + - name: First arguments after the lists must be string set + set_fact: + my_list: "{{ list1 | community.general.lists_mergeby(list2, {'x': 'y'}) }}" + register: result + ignore_errors: true + - name: First arguments after the lists must be string debug + debug: + var: my_list + when: debug_test|d(false)|bool + - name: First arguments after the lists must be string assert + assert: + that: + - result is failed + - '"First argument after the lists for community.general.lists_mergeby must be string." in result.msg' + tags: t6 + +- name: 7.Elements of list must be dictionaries + block: + - name: Elements of list must be dictionaries set + set_fact: + my_list: "{{ list4 | community.general.lists_mergeby(list2, 'name') }}" + register: result + ignore_errors: true + - name: Elements of list must be dictionaries debug + debug: + var: my_list + when: debug_test|d(false)|bool + - name: Elements of list must be dictionaries assert + assert: + that: + - result is failed + - '"Elements of list arguments for lists_mergeby must be dictionaries." in result.msg' + tags: t7 + +- name: 8.Merge 3 lists by attribute name. 1 list in params. + block: + - name: Merge 3 lists by attribute name. 1 list in params. set + set_fact: + my_list: "{{ [list1, list2] | community.general.lists_mergeby(list5, 'name') }}" + - name: Merge 3 lists by attribute name. 1 list in params. debug + debug: + var: my_list + when: debug_test|d(false)|bool + - name: Merge 3 lists by attribute name. 1 list in params. assert + assert: + that: my_list | difference(result1) | length == 0 + tags: t8 + +- name: 9.Merge 3 lists by attribute name. No list in the params. + block: + - name: Merge 3 lists by attribute name. No list in the params. set + set_fact: + my_list: "{{ [list1, list2, list5] | community.general.lists_mergeby('name') }}" + - name: Merge 3 lists by attribute name. No list in the params. debug + debug: + var: my_list + when: debug_test|d(false)|bool + - name: Merge 3 lists by attribute name. No list in the params. asset + assert: + that: my_list | difference(result1) | length == 0 + tags: t9 + +# Test list_merge default options + +- name: 100.Merge 2 lists by attribute name. list_merge='replace' + block: + - name: Merge 2 lists by attribute name. list_merge='replace'. set + set_fact: + my_list: "{{ [list100, list101] | community.general.lists_mergeby('name') }}" + - name: Merge 2 lists by attribute name. list_merge='replace'. debug + debug: + msg: |- + my_list: + {{ my_list|to_nice_yaml|indent(2) }} + my_list|difference(result100): + {{ my_list|difference(result100)|to_nice_yaml|indent(2) }} + when: debug_test|d(false)|bool + - name: Merge 2 lists by attribute name. list_merge='replace'. assert + assert: + that: my_list | difference(result100) | length == 0 + tags: t100 diff --git a/tests/integration/targets/filter_list/tasks/main.yml b/tests/integration/targets/filter_list/tasks/main.yml index 1e5773c638..87f0e350c4 100644 --- a/tests/integration/targets/filter_list/tasks/main.yml +++ b/tests/integration/targets/filter_list/tasks/main.yml @@ -1,2 +1,7 @@ --- -- include_tasks: lists_mergeby.yml +- name: Test list_merge default options + import_tasks: lists_mergeby_default.yml + +- name: Test list_merge non-default options in Ansible 2.10 and higher + import_tasks: lists_mergeby_2-10.yml + when: ansible_version.full is version('2.10', '>=') diff --git a/tests/integration/targets/filter_list/vars/main.yml b/tests/integration/targets/filter_list/vars/main.yml index 13887e176c..83d831f278 100644 --- a/tests/integration/targets/filter_list/vars/main.yml +++ b/tests/integration/targets/filter_list/vars/main.yml @@ -1,3 +1,4 @@ +--- list1: - name: myname01 param01: myparam01 @@ -25,3 +26,180 @@ list4: - name: myname01 param01: myparam01 - myname02 + +list5: + - name: myname01 + param01: myparam05 + - name: myname02 + param01: myparam06 + +result1: + - name: myname01 + param01: myparam05 + - name: myname02 + param01: myparam06 + param02: myparam04 + - name: myname03 + param03: myparam03 + +# Test list_merge + +list100: + - name: myname01 + param01: + - default1 + - name: myname02 + param01: + - default2 + +list101: + - name: myname01 + param01: + - patch1 + - name: myname02 + param01: + - patch2 + +list102: + - name: myname01 + param01: + - patch1a + - patch1b + - patch1c + - name: myname02 + param01: + - patch2a + - patch2b + - patch2d + +list103: + - name: myname01 + param01: + - patch1c + - patch1d + - name: myname02 + param01: + - patch2c + - patch2d + +result100: + - name: myname01 + param01: + - patch1 + - name: myname02 + param01: + - patch2 + +result101: + - name: myname01 + param01: + - default1 + - name: myname02 + param01: + - default2 + +result102: + - name: myname01 + param01: + - default1 + - patch1 + - name: myname02 + param01: + - default2 + - patch2 + +result103: + - name: myname01 + param01: + - patch1 + - default1 + - name: myname02 + param01: + - patch2 + - default2 + +result104: + - name: myname01 + param01: + - patch1a + - patch1b + - patch1c + - patch1d + - name: myname02 + param01: + - patch2a + - patch2b + - patch2c + - patch2d + +result105: + - name: myname01 + param01: + - patch1c + - patch1d + - patch1a + - patch1b + - name: myname02 + param01: + - patch2c + - patch2d + - patch2a + - patch2b + +# Test recursive + +list200: + - name: myname01 + param01: + x: default_value + y: default_value + list: + - default_value + - name: myname02 + param01: [1, 1, 2, 3] + +list201: + - name: myname01 + param01: + y: patch_value + z: patch_value + list: + - patch_value + - name: myname02 + param01: [3, 4, 4, {key: value}] + +result200: + - name: myname01 + param01: + list: + - default_value + - patch_value + x: default_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + - 4 + - 4 + - key: value + +result201: + - name: myname01 + param01: + list: + - patch_value + y: patch_value + z: patch_value + - name: myname02 + param01: + - 1 + - 1 + - 2 + - 3 + - 4 + - 4 + - key: value