diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 8f86d5fe2a..dc1ad3989c 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -151,8 +151,18 @@ files: $filters/jc.py: maintainers: kellyjonbrazil $filters/json_query.py: {} + $filters/lists.py: + maintainers: cfiehe + $filters/lists_difference.yml: + maintainers: cfiehe + $filters/lists_intersect.yml: + maintainers: cfiehe $filters/lists_mergeby.py: maintainers: vbotka + $filters/lists_symmetric_difference.yml: + maintainers: cfiehe + $filters/lists_union.yml: + maintainers: cfiehe $filters/random_mac.py: {} $filters/time.py: maintainers: resmo @@ -1442,6 +1452,8 @@ files: maintainers: felixfontein giner docs/docsite/rst/filter_guide_abstract_informations_grouping.rst: maintainers: felixfontein + docs/docsite/rst/filter_guide_abstract_informations_lists_helper.rst: + maintainers: cfiehe docs/docsite/rst/filter_guide_abstract_informations_merging_lists_of_dictionaries.rst: maintainers: vbotka docs/docsite/rst/filter_guide_conversions.rst: diff --git a/docs/docsite/rst/filter_guide_abstract_informations.rst b/docs/docsite/rst/filter_guide_abstract_informations.rst index 8f997f1637..cac85089a0 100644 --- a/docs/docsite/rst/filter_guide_abstract_informations.rst +++ b/docs/docsite/rst/filter_guide_abstract_informations.rst @@ -12,4 +12,5 @@ Abstract transformations filter_guide_abstract_informations_dictionaries filter_guide_abstract_informations_grouping filter_guide_abstract_informations_merging_lists_of_dictionaries + filter_guide_abstract_informations_lists_helper filter_guide_abstract_informations_counting_elements_in_sequence diff --git a/docs/docsite/rst/filter_guide_abstract_informations_lists_helper.rst b/docs/docsite/rst/filter_guide_abstract_informations_lists_helper.rst new file mode 100644 index 0000000000..63db10a782 --- /dev/null +++ b/docs/docsite/rst/filter_guide_abstract_informations_lists_helper.rst @@ -0,0 +1,81 @@ +.. + Copyright (c) Ansible Project + GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) + SPDX-License-Identifier: GPL-3.0-or-later + +Union, intersection and difference of lists +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Starting with Ansible Core 2.16, the builtin filters :ansplugin:`ansible.builtin.union#filter`, :ansplugin:`ansible.builtin.intersect#filter`, :ansplugin:`ansible.builtin.difference#filter` and :ansplugin:`ansible.builtin.symmetric_difference#filter` began to behave differently and do no longer preserve the item order. Items in the resulting lists are returned in arbitrary order and the order can vary between subsequent runs. + +The Ansible community.general collection provides the following additional list filters: + +- :ansplugin:`community.general.lists_union#filter` +- :ansplugin:`community.general.lists_intersect#filter` +- :ansplugin:`community.general.lists_difference#filter` +- :ansplugin:`community.general.lists_symmetric_difference#filter` + +These filters preserve the item order, eliminate duplicates and are an extended version of the builtin ones, because they can operate on more than two lists. + +.. note:: Stick to the builtin filters, when item order is not important or when you do not need the n-ary operating mode. The builtin filters are faster, because they rely mostly on sets as their underlying datastructure. + +Let us use the lists below in the following examples: + +.. code-block:: yaml + + A: [9, 5, 7, 1, 9, 4, 10, 5, 9, 7] + B: [4, 1, 2, 8, 3, 1, 7] + C: [10, 2, 1, 9, 1] + +The union of ``A`` and ``B`` can be written as: + +.. code-block:: yaml+jinja + + result: "{{ A | community.general.lists_union(B) }}" + +This statement produces: + +.. code-block:: yaml + + result: [9, 5, 7, 1, 4, 10, 2, 8, 3] + +If you want to calculate the intersection of ``A``, ``B`` and ``C``, you can use the following statement: + +.. code-block:: yaml+jinja + + result: "{{ A | community.general.lists_intersect(B, C) }}" + +Alternatively, you can use a list of lists as an input of the filter + +.. code-block:: yaml+jinja + + result: "{{ [A, B] | community.general.lists_intersect(C) }}" + +or + +.. code-block:: yaml+jinja + + result: "{{ [A, B, C] | community.general.lists_intersect(flatten=true) }}" + +All three statements are equivalent and give: + +.. code-block:: yaml + + result: [1] + +.. note:: Be aware that in most cases, filter calls without any argument require ``flatten=true``, otherwise the input is returned as result. The reason for this is, that the input is considered as a variable argument and is wrapped by an additional outer list. ``flatten=true`` ensures that this list is removed before the input is processed by the filter logic. + +The filters ansplugin:`community.general.lists_difference#filter` or :ansplugin:`community.general.lists_symmetric_difference#filter` can be used in the same way as the filters in the examples above. They calculate the difference or the symmetric difference between two or more lists and preserve the item order. + +For example, the symmetric difference of ``A``, ``B`` and ``C`` may be written as: + +.. code-block:: yaml+jinja + + result: "{{ A | community.general.lists_symmetric_difference(B, C) }}" + +This gives: + +.. code-block:: yaml + + result: [5, 8, 3, 1] + diff --git a/plugins/filter/lists.py b/plugins/filter/lists.py new file mode 100644 index 0000000000..d16f955c22 --- /dev/null +++ b/plugins/filter/lists.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from ansible.errors import AnsibleFilterError +from ansible.module_utils.common.collections import is_sequence + + +def remove_duplicates(lst): + seen = set() + seen_add = seen.add + result = [] + for item in lst: + try: + if item not in seen: + seen_add(item) + result.append(item) + except TypeError: + # This happens for unhashable values `item`. If this happens, + # convert `seen` to a list and continue. + seen = list(seen) + seen_add = seen.append + if item not in seen: + seen_add(item) + result.append(item) + return result + + +def flatten_list(lst): + result = [] + for sublist in lst: + if not is_sequence(sublist): + msg = ("All arguments must be lists. %s is %s") + raise AnsibleFilterError(msg % (sublist, type(sublist))) + if len(sublist) > 0: + if all(is_sequence(sub) for sub in sublist): + for item in sublist: + result.append(item) + else: + result.append(sublist) + return result + + +def lists_union(*args, **kwargs): + lists = args + flatten = kwargs.pop('flatten', False) + + if kwargs: + # Some unused kwargs remain + raise AnsibleFilterError( + "lists_union() got unexpected keywords arguments: {0}".format( + ", ".join(kwargs.keys()) + ) + ) + + if flatten: + lists = flatten_list(args) + + if not lists: + return [] + + if len(lists) == 1: + return lists[0] + + a = lists[0] + for b in lists[1:]: + a = do_union(a, b) + return remove_duplicates(a) + + +def do_union(a, b): + return a + b + + +def lists_intersect(*args, **kwargs): + lists = args + flatten = kwargs.pop('flatten', False) + + if kwargs: + # Some unused kwargs remain + raise AnsibleFilterError( + "lists_intersect() got unexpected keywords arguments: {0}".format( + ", ".join(kwargs.keys()) + ) + ) + + if flatten: + lists = flatten_list(args) + + if not lists: + return [] + + if len(lists) == 1: + return lists[0] + + a = remove_duplicates(lists[0]) + for b in lists[1:]: + a = do_intersect(a, b) + return a + + +def do_intersect(a, b): + isect = [] + try: + other = set(b) + isect = [item for item in a if item in other] + except TypeError: + # This happens for unhashable values, + # use a list instead and redo. + other = list(b) + isect = [item for item in a if item in other] + return isect + + +def lists_difference(*args, **kwargs): + lists = args + flatten = kwargs.pop('flatten', False) + + if kwargs: + # Some unused kwargs remain + raise AnsibleFilterError( + "lists_difference() got unexpected keywords arguments: {0}".format( + ", ".join(kwargs.keys()) + ) + ) + + if flatten: + lists = flatten_list(args) + + if not lists: + return [] + + if len(lists) == 1: + return lists[0] + + a = remove_duplicates(lists[0]) + for b in lists[1:]: + a = do_difference(a, b) + return a + + +def do_difference(a, b): + diff = [] + try: + other = set(b) + diff = [item for item in a if item not in other] + except TypeError: + # This happens for unhashable values, + # use a list instead and redo. + other = list(b) + diff = [item for item in a if item not in other] + return diff + + +def lists_symmetric_difference(*args, **kwargs): + lists = args + flatten = kwargs.pop('flatten', False) + + if kwargs: + # Some unused kwargs remain + raise AnsibleFilterError( + "lists_difference() got unexpected keywords arguments: {0}".format( + ", ".join(kwargs.keys()) + ) + ) + + if flatten: + lists = flatten_list(args) + + if not lists: + return [] + + if len(lists) == 1: + return lists[0] + + a = lists[0] + for b in lists[1:]: + a = do_symmetric_difference(a, b) + return a + + +def do_symmetric_difference(a, b): + sym_diff = [] + union = lists_union(a, b) + try: + isect = set(a) & set(b) + sym_diff = [item for item in union if item not in isect] + except TypeError: + # This happens for unhashable values, + # build the intersection of `a` and `b` backed + # by a list instead of a set and redo. + isect = lists_intersect(a, b) + sym_diff = [item for item in union if item not in isect] + return sym_diff + + +class FilterModule(object): + ''' Ansible lists jinja2 filters ''' + + def filters(self): + return { + 'lists_union': lists_union, + 'lists_intersect': lists_intersect, + 'lists_difference': lists_difference, + 'lists_symmetric_difference': lists_symmetric_difference, + } diff --git a/plugins/filter/lists_difference.yml b/plugins/filter/lists_difference.yml new file mode 100644 index 0000000000..9806a9f0bc --- /dev/null +++ b/plugins/filter/lists_difference.yml @@ -0,0 +1,48 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: lists_difference + short_description: Difference of lists with a predictive order + version_added: 8.4.0 + description: + - Provide a unique list of all the elements from the first which do not appear in the other lists. + - The order of the items in the resulting list is preserved. + options: + _input: + description: A list. + type: list + elements: any + required: true + flatten: + description: Whether to remove one hierarchy level from the input list. + type: boolean + default: false + author: + - Christoph Fiehe (@cfiehe) + +EXAMPLES: | + - name: Return the difference of list1 and list2. + ansible.builtin.debug: + msg: "{{ list1 | community.general.lists_difference(list2) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + # => [10] + + - name: Return the difference of list1, list2 and list3. + ansible.builtin.debug: + msg: "{{ [list1, list2, list3] | community.general.lists_difference(flatten=true) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + list3: [1, 2, 3, 4, 5, 10, 99, 101] + # => [] + +RETURN: + _value: + description: A unique list of all the elements from the first list that do not appear on the other lists. + type: list + elements: any diff --git a/plugins/filter/lists_intersect.yml b/plugins/filter/lists_intersect.yml new file mode 100644 index 0000000000..8253463dee --- /dev/null +++ b/plugins/filter/lists_intersect.yml @@ -0,0 +1,48 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: lists_intersect + short_description: Intersection of lists with a predictive order + version_added: 8.4.0 + description: + - Provide a unique list of all the common elements of two or more lists. + - The order of the items in the resulting list is preserved. + options: + _input: + description: A list. + type: list + elements: any + required: true + flatten: + description: Whether to remove one hierarchy level from the input list. + type: boolean + default: false + author: + - Christoph Fiehe (@cfiehe) + +EXAMPLES: | + - name: Return the intersection of list1 and list2. + ansible.builtin.debug: + msg: "{{ list1 | community.general.lists_intersect(list2) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + # => [1, 2, 5, 3, 4] + + - name: Return the intersection of list1, list2 and list3. + ansible.builtin.debug: + msg: "{{ [list1, list2, list3] | community.general.lists_intersect(flatten=true) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + list3: [1, 2, 3, 4, 5, 10, 99, 101] + # => [1, 2, 5, 3, 4] + +RETURN: + _value: + description: A unique list of all the common elements from the provided lists. + type: list + elements: any diff --git a/plugins/filter/lists_symmetric_difference.yml b/plugins/filter/lists_symmetric_difference.yml new file mode 100644 index 0000000000..d985704c2c --- /dev/null +++ b/plugins/filter/lists_symmetric_difference.yml @@ -0,0 +1,48 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: lists_symmetric_difference + short_description: Symmetric Difference of lists with a predictive order + version_added: 8.4.0 + description: + - Provide a unique list containing the symmetric difference of two or more lists. + - The order of the items in the resulting list is preserved. + options: + _input: + description: A list. + type: list + elements: any + required: true + flatten: + description: Whether to remove one hierarchy level from the input list. + type: boolean + default: false + author: + - Christoph Fiehe (@cfiehe) + +EXAMPLES: | + - name: Return the symmetric difference of list1 and list2. + ansible.builtin.debug: + msg: "{{ list1 | community.general.lists_symmetric_difference(list2) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + # => [10, 11, 99] + + - name: Return the symmetric difference of list1, list2 and list3. + ansible.builtin.debug: + msg: "{{ [list1, list2, list3] | community.general.lists_symmetric_difference(flatten=true) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + list3: [1, 2, 3, 4, 5, 10, 99, 101] + # => [11, 1, 2, 3, 4, 5, 101] + +RETURN: + _value: + description: A unique list containing the symmetric difference of two or more lists. + type: list + elements: any diff --git a/plugins/filter/lists_union.yml b/plugins/filter/lists_union.yml new file mode 100644 index 0000000000..ba69090836 --- /dev/null +++ b/plugins/filter/lists_union.yml @@ -0,0 +1,48 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: lists_union + short_description: Union of lists with a predictive order + version_added: 8.4.0 + description: + - Provide a unique list of all the elements of two or more lists. + - The order of the items in the resulting list is preserved. + options: + _input: + description: A list. + type: list + elements: any + required: true + flatten: + description: Whether to remove one hierarchy level from the input list. + type: boolean + default: false + author: + - Christoph Fiehe (@cfiehe) + +EXAMPLES: | + - name: Return the union of list1, list2 and list3. + ansible.builtin.debug: + msg: "{{ list1 | community.general.lists_union(list2, list3) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + list3: [1, 2, 3, 4, 5, 10, 99, 101] + # => [1, 2, 5, 3, 4, 10, 11, 99, 101] + + - name: Return the union of list1 and list2. + ansible.builtin.debug: + msg: "{{ [list1, list2] | community.general.lists_union(flatten=true) }}" + vars: + list1: [1, 2, 5, 3, 4, 10] + list2: [1, 2, 3, 4, 5, 11, 99] + # => [1, 2, 5, 3, 4, 10, 11, 99] + +RETURN: + _value: + description: A unique list of all the elements from the provided lists. + type: list + elements: any diff --git a/tests/integration/targets/filter_lists/aliases b/tests/integration/targets/filter_lists/aliases new file mode 100644 index 0000000000..12d1d6617e --- /dev/null +++ b/tests/integration/targets/filter_lists/aliases @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/2 diff --git a/tests/integration/targets/filter_lists/tasks/main.yml b/tests/integration/targets/filter_lists/tasks/main.yml new file mode 100644 index 0000000000..a146e1293e --- /dev/null +++ b/tests/integration/targets/filter_lists/tasks/main.yml @@ -0,0 +1,64 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Test predictive list union + ansible.builtin.assert: + that: + - 'list1 | community.general.lists_union(list2, list3) == [1, 2, 5, 3, 4, 10, 11, 99, 101]' + - '[list1, list2, list3] | community.general.lists_union(flatten=True) == [1, 2, 5, 3, 4, 10, 11, 99, 101]' + - '[1, 2, 3] | community.general.lists_union([4, 5, 6]) == [1, 2, 3, 4, 5, 6]' + - '[1, 2, 3] | community.general.lists_union([3, 4, 5, 6]) == [1, 2, 3, 4, 5, 6]' + - '[1, 2, 3] | community.general.lists_union([3, 2, 1]) == [1, 2, 3]' + - '["a", "A", "b"] | community.general.lists_union(["B", "c", "C"]) == ["a", "A", "b", "B", "c", "C"]' + - '["a", "A", "b"] | community.general.lists_union(["b", "B", "c", "C"]) == ["a", "A", "b", "B", "c", "C"]' + - '["a", "A", "b"] | community.general.lists_union(["b", "A", "a"]) == ["a", "A", "b"]' + - '[["a"]] | community.general.lists_union([["b"], ["a"]]) == [["a"], ["b"]]' + - '[["a"]] | community.general.lists_union([["b"]], ["a"]) == [["a"], ["b"], "a"]' + - '[["a"]] | community.general.lists_union(["b"], ["a"]) == [["a"], "b", "a"]' + +- name: Test predictive list intersection + ansible.builtin.assert: + that: + - 'list1 | community.general.lists_intersect(list2, list3) == [1, 2, 5, 4]' + - '[list1, list2, list3] | community.general.lists_intersect(flatten=True) == [1, 2, 5, 4]' + - '[1, 2, 3] | community.general.lists_intersect([4, 5, 6]) == []' + - '[1, 2, 3] | community.general.lists_intersect([3, 4, 5, 6]) == [3]' + - '[1, 2, 3] | community.general.lists_intersect([3, 2, 1]) == [1, 2, 3]' + - '["a", "A", "b"] | community.general.lists_intersect(["B", "c", "C"]) == []' + - '["a", "A", "b"] | community.general.lists_intersect(["b", "B", "c", "C"]) == ["b"]' + - '["a", "A", "b"] | community.general.lists_intersect(["b", "A", "a"]) == ["a", "A", "b"]' + - '[["a"]] | community.general.lists_intersect([["b"], ["a"]]) == [["a"]]' + - '[["a"]] | community.general.lists_intersect([["b"]], ["a"]) == []' + - '[["a"]] | community.general.lists_intersect(["b"], ["a"]) == []' + +- name: Test predictive list difference + ansible.builtin.assert: + that: + - 'list1 | community.general.lists_difference(list2, list3) == []' + - '[list1, list2, list3] | community.general.lists_difference(flatten=True) == []' + - '[1, 2, 3] | community.general.lists_difference([4, 5, 6]) == [1, 2, 3]' + - '[1, 2, 3] | community.general.lists_difference([3, 4, 5, 6]) == [1, 2]' + - '[1, 2, 3] | community.general.lists_difference([3, 2, 1]) == []' + - '["a", "A", "b"] | community.general.lists_difference(["B", "c", "C"]) == ["a", "A", "b"]' + - '["a", "A", "b"] | community.general.lists_difference(["b", "B", "c", "C"]) == ["a", "A"]' + - '["a", "A", "b"] | community.general.lists_difference(["b", "A", "a"]) == []' + - '[["a"]] | community.general.lists_difference([["b"], ["a"]]) == []' + - '[["a"]] | community.general.lists_difference([["b"]], ["a"]) == [["a"]]' + - '[["a"]] | community.general.lists_difference(["b"], ["a"]) == [["a"]]' + +- name: Test predictive list symmetric difference + ansible.builtin.assert: + that: + - 'list1 | community.general.lists_symmetric_difference(list2, list3) == [11, 1, 2, 4, 5, 101]' + - '[list1, list2, list3] | community.general.lists_symmetric_difference(flatten=True) == [11, 1, 2, 4, 5, 101]' + - '[1, 2, 3] | community.general.lists_symmetric_difference([4, 5, 6]) == [1, 2, 3, 4, 5, 6]' + - '[1, 2, 3] | community.general.lists_symmetric_difference([3, 4, 5, 6]) == [1, 2, 4, 5, 6]' + - '[1, 2, 3] | community.general.lists_symmetric_difference([3, 2, 1]) == []' + - '["a", "A", "b"] | community.general.lists_symmetric_difference(["B", "c", "C"]) == ["a", "A", "b", "B", "c", "C"]' + - '["a", "A", "b"] | community.general.lists_symmetric_difference(["b", "B", "c", "C"]) == ["a", "A", "B", "c", "C"]' + - '["a", "A", "b"] | community.general.lists_symmetric_difference(["b", "A", "a"]) == []' + - '[["a"]] | community.general.lists_symmetric_difference([["b"], ["a"]]) == [["b"]]' + - '[["a"]] | community.general.lists_symmetric_difference([["b"]], ["a"]) == [["a"], ["b"], "a"]' + - '[["a"]] | community.general.lists_symmetric_difference(["b"], ["a"]) == [["a"], "b", "a"]' diff --git a/tests/integration/targets/filter_lists/vars/main.yml b/tests/integration/targets/filter_lists/vars/main.yml new file mode 100644 index 0000000000..a67af1dadb --- /dev/null +++ b/tests/integration/targets/filter_lists/vars/main.yml @@ -0,0 +1,8 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +list1: [1, 2, 5, 3, 4, 10] +list2: [1, 2, 3, 4, 5, 11, 99] +list3: [1, 2, 4, 5, 10, 99, 101]