From 5b10f8234cfb11158a8b6eb5be895c0caa1ab56d Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Mon, 1 Nov 2021 19:01:52 +0100 Subject: [PATCH] Add collection_version lookup (#3633) * Add collection_verison lookup plugin. * Document ansible-base 2.10 and ansible-core 2.11 special case. * Add default for result_not_found. * Simplify not existing test. * Move metadata loading to function. * Update plugins/lookup/collection_version.py Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com> * Extend tests. * Fix tests for ansible-core < 2.12. Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com> --- .github/BOTMETA.yml | 2 + plugins/lookup/collection_version.py | 138 ++++++++++++++++++ .../targets/lookup_collection_version/aliases | 1 + .../testns/testcoll/galaxy.yml | 7 + .../plugins/modules/collection_module.py | 32 ++++ .../testns/testcoll_mf/FILES.json | 40 +++++ .../testns/testcoll_mf/MANIFEST.json | 30 ++++ .../testns/testcoll_mf/README.md | 0 .../plugins/modules/collection_module.py | 32 ++++ .../plugins/modules/collection_module.py | 32 ++++ .../testns/testcoll_nv/galaxy.yml | 6 + .../plugins/modules/collection_module.py | 32 ++++ .../library/local_module.py | 32 ++++ .../lookup_collection_version/runme.sh | 18 +++ .../lookup_collection_version/runme.yml | 35 +++++ 15 files changed, 437 insertions(+) create mode 100644 plugins/lookup/collection_version.py create mode 100644 tests/integration/targets/lookup_collection_version/aliases create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/galaxy.yml create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/plugins/modules/collection_module.py create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/FILES.json create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/MANIFEST.json create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/README.md create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/plugins/modules/collection_module.py create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nothing/plugins/modules/collection_module.py create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/galaxy.yml create mode 100644 tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/plugins/modules/collection_module.py create mode 100644 tests/integration/targets/lookup_collection_version/library/local_module.py create mode 100755 tests/integration/targets/lookup_collection_version/runme.sh create mode 100644 tests/integration/targets/lookup_collection_version/runme.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 1f7be1f81e..a497c3daa2 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -175,6 +175,8 @@ files: labels: lookups $lookups/cartesian.py: {} $lookups/chef_databag.py: {} + $lookups/collection_version.py: + maintainers: felixfontein $lookups/consul_kv.py: {} $lookups/credstash.py: {} $lookups/cyberarkpassword.py: diff --git a/plugins/lookup/collection_version.py b/plugins/lookup/collection_version.py new file mode 100644 index 0000000000..bb67b3b153 --- /dev/null +++ b/plugins/lookup/collection_version.py @@ -0,0 +1,138 @@ +# (c) 2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = """ +name: collection_version +author: Felix Fontein (@felixfontein) +version_added: "4.0.0" +short_description: Retrieves the version of an installed collection +description: + - This lookup allows to query the version of an installed collection, and to determine whether a + collection is installed at all. + - By default it returns C(none) for non-existing collections and C(*) for collections without a + version number. The latter should only happen in development environments, or when installing + a collection from git which has no version in its C(galaxy.yml). This behavior can be adjusted + by providing other values with I(result_not_found) and I(result_no_version). +options: + _terms: + description: + - The collections to look for. + - For example C(community.general). + type: list + elements: str + required: true + result_not_found: + description: + - The value to return when the collection could not be found. + - By default, C(none) is returned. + type: string + default: ~ + result_no_version: + description: + - The value to return when the collection has no version number. + - This can happen for collections installed from git which do not have a version number + in C(galaxy.yml). + - By default, C(*) is returned. + type: string + default: '*' +""" + +EXAMPLES = """ +- name: Check version of community.general + ansible.builtin.debug: + msg: "community.general version {{ lookup('community.general.collection_version', 'community.general') }}" +""" + +RETURN = """ + _raw: + description: + - The version number of the collections listed as input. + - If a collection can not be found, it will return the value provided in I(result_not_found). + By default, this is C(none). + - If a collection can be found, but the version not identified, it will return the value provided in + I(result_no_version). By default, this is C(*). This can happen for collections installed + from git which do not have a version number in C(galaxy.yml). + type: list + elements: str +""" + +import json +import os +import re + +import yaml + +from ansible.errors import AnsibleLookupError +from ansible.module_utils.compat.importlib import import_module +from ansible.plugins.lookup import LookupBase + + +FQCN_RE = re.compile(r'^[A-Za-z0-9_]+\.[A-Za-z0-9_]+$') + + +def load_collection_meta_manifest(manifest_path): + with open(manifest_path, 'rb') as f: + meta = json.load(f) + return { + 'version': meta['collection_info']['version'], + } + + +def load_collection_meta_galaxy(galaxy_path, no_version='*'): + with open(galaxy_path, 'rb') as f: + meta = yaml.safe_load(f) + return { + 'version': meta.get('version') or no_version, + } + + +def load_collection_meta(collection_pkg, no_version='*'): + path = os.path.dirname(collection_pkg.__file__) + + # Try to load MANIFEST.json + manifest_path = os.path.join(path, 'MANIFEST.json') + if os.path.exists(manifest_path): + return load_collection_meta_manifest(manifest_path) + + # Try to load galaxy.y(a)ml + galaxy_path = os.path.join(path, 'galaxy.yml') + galaxy_alt_path = os.path.join(path, 'galaxy.yaml') + # galaxy.yaml was only supported in ansible-base 2.10 and ansible-core 2.11. Support was removed + # in https://github.com/ansible/ansible/commit/595413d11346b6f26bb3d9df2d8e05f2747508a3 for + # ansible-core 2.12. + for path in (galaxy_path, galaxy_alt_path): + if os.path.exists(path): + return load_collection_meta_galaxy(path, no_version=no_version) + + return {} + + +class LookupModule(LookupBase): + def run(self, terms, variables=None, **kwargs): + result = [] + self.set_options(var_options=variables, direct=kwargs) + not_found = self.get_option('result_not_found') + no_version = self.get_option('result_no_version') + + for term in terms: + if not FQCN_RE.match(term): + raise AnsibleLookupError('"{term}" is not a FQCN'.format(term=term)) + + try: + collection_pkg = import_module('ansible_collections.{fqcn}'.format(fqcn=term)) + except ImportError: + # Collection not found + result.append(not_found) + continue + + try: + data = load_collection_meta(collection_pkg, no_version=no_version) + except Exception as exc: + raise AnsibleLookupError('Error while loading metadata for {fqcn}: {error}'.format(fqcn=term, error=exc)) + + result.append(data.get('version', no_version)) + + return result diff --git a/tests/integration/targets/lookup_collection_version/aliases b/tests/integration/targets/lookup_collection_version/aliases new file mode 100644 index 0000000000..b59832142f --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/aliases @@ -0,0 +1 @@ +shippable/posix/group3 diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/galaxy.yml b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/galaxy.yml new file mode 100644 index 0000000000..f66a8af4c9 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/galaxy.yml @@ -0,0 +1,7 @@ +namespace: testns +name: testcoll +version: 0.0.1 +authors: + - Ansible (https://github.com/ansible) +description: null +tags: [community] diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/plugins/modules/collection_module.py b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/plugins/modules/collection_module.py new file mode 100644 index 0000000000..8db51c39a7 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/plugins/modules/collection_module.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +module: collection_module +short_description: Test collection module +description: + - This is a test module in a local collection. +author: "Felix Fontein (@felixfontein)" +options: {} +''' + +EXAMPLES = ''' # ''' + +RETURN = ''' # ''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + AnsibleModule(argument_spec={}).exit_json() + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/FILES.json b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/FILES.json new file mode 100644 index 0000000000..99eb4c0bfb --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/FILES.json @@ -0,0 +1,40 @@ +{ + "files": [ + { + "name": ".", + "ftype": "dir", + "chksum_type": null, + "chksum_sha256": null, + "format": 1 + }, + { + "name": "README.md", + "ftype": "file", + "chksum_type": "sha256", + "chksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "format": 1 + }, + { + "name": "plugins", + "ftype": "dir", + "chksum_type": null, + "chksum_sha256": null, + "format": 1 + }, + { + "name": "plugins/modules", + "ftype": "dir", + "chksum_type": null, + "chksum_sha256": null, + "format": 1 + }, + { + "name": "plugins/modules/collection_module.py", + "ftype": "file", + "chksum_type": "sha256", + "chksum_sha256": "c3f0114d080c409c58c8846be8da7b91137b38eaf2d24f72a4a61a303f925f4d", + "format": 1 + } + ], + "format": 1 +} \ No newline at end of file diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/MANIFEST.json b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/MANIFEST.json new file mode 100644 index 0000000000..f1d009a73f --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/MANIFEST.json @@ -0,0 +1,30 @@ +{ + "collection_info": { + "namespace": "testns", + "name": "testcoll_mf", + "version": "0.0.1", + "authors": [ + "Ansible (https://github.com/ansible)" + ], + "readme": "README.md", + "tags": [ + "community" + ], + "description": null, + "license": [], + "license_file": null, + "dependencies": {}, + "repository": null, + "documentation": null, + "homepage": null, + "issues": null + }, + "file_manifest_file": { + "name": "FILES.json", + "ftype": "file", + "chksum_type": "sha256", + "chksum_sha256": "025818f18fcae5c9f78d778ae6e246ecffed6d56a886ffbc145cb66d54e9951e", + "format": 1 + }, + "format": 1 +} \ No newline at end of file diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/README.md b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/plugins/modules/collection_module.py b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/plugins/modules/collection_module.py new file mode 100644 index 0000000000..8db51c39a7 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/plugins/modules/collection_module.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +module: collection_module +short_description: Test collection module +description: + - This is a test module in a local collection. +author: "Felix Fontein (@felixfontein)" +options: {} +''' + +EXAMPLES = ''' # ''' + +RETURN = ''' # ''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + AnsibleModule(argument_spec={}).exit_json() + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nothing/plugins/modules/collection_module.py b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nothing/plugins/modules/collection_module.py new file mode 100644 index 0000000000..8db51c39a7 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nothing/plugins/modules/collection_module.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +module: collection_module +short_description: Test collection module +description: + - This is a test module in a local collection. +author: "Felix Fontein (@felixfontein)" +options: {} +''' + +EXAMPLES = ''' # ''' + +RETURN = ''' # ''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + AnsibleModule(argument_spec={}).exit_json() + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/galaxy.yml b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/galaxy.yml new file mode 100644 index 0000000000..599a47e649 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/galaxy.yml @@ -0,0 +1,6 @@ +namespace: testns +name: testcoll_nv +authors: + - Ansible (https://github.com/ansible) +description: null +tags: [community] diff --git a/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/plugins/modules/collection_module.py b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/plugins/modules/collection_module.py new file mode 100644 index 0000000000..8db51c39a7 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/plugins/modules/collection_module.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +module: collection_module +short_description: Test collection module +description: + - This is a test module in a local collection. +author: "Felix Fontein (@felixfontein)" +options: {} +''' + +EXAMPLES = ''' # ''' + +RETURN = ''' # ''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + AnsibleModule(argument_spec={}).exit_json() + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/lookup_collection_version/library/local_module.py b/tests/integration/targets/lookup_collection_version/library/local_module.py new file mode 100644 index 0000000000..424c127ced --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/library/local_module.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2021, Felix Fontein +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +module: local_module +short_description: Test local module +description: + - This is a test module locally next to a playbook. +author: "Felix Fontein (@felixfontein)" +options: {} +''' + +EXAMPLES = ''' # ''' + +RETURN = ''' # ''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + AnsibleModule(argument_spec={}).exit_json() + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/lookup_collection_version/runme.sh b/tests/integration/targets/lookup_collection_version/runme.sh new file mode 100755 index 0000000000..8ce47c86d0 --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/runme.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -eux + +export ANSIBLE_TEST_PREFER_VENV=1 # see https://github.com/ansible/ansible/pull/73000#issuecomment-757012395; can be removed once Ansible 2.9 and ansible-base 2.10 support has been dropped +source virtualenv.sh + +# The collection loader ignores paths which have more than one ansible_collections in it. +# That's why we have to copy this directory to a temporary place and run the test there. + +# Create temporary folder +TEMPDIR=$(mktemp -d) +trap '{ rm -rf ${TEMPDIR}; }' EXIT + +cp -r . "${TEMPDIR}" +cd "${TEMPDIR}" + +ansible-playbook runme.yml "$@" diff --git a/tests/integration/targets/lookup_collection_version/runme.yml b/tests/integration/targets/lookup_collection_version/runme.yml new file mode 100644 index 0000000000..aed52bee2d --- /dev/null +++ b/tests/integration/targets/lookup_collection_version/runme.yml @@ -0,0 +1,35 @@ +- hosts: localhost + tasks: + - name: Test collection_version + assert: + that: + # Collection that does not exist + - query('community.general.collection_version', 'foo.bar') == [none] + - lookup('community.general.collection_version', 'foo.bar', result_not_found='foo') == 'foo' + # Collection that exists + - lookup('community.general.collection_version', 'community.general') is string + # Local collection + - lookup('community.general.collection_version', 'testns.testcoll') == '0.0.1' + # Local collection with no version + - lookup('community.general.collection_version', 'testns.testcoll_nv') == '*' + - lookup('community.general.collection_version', 'testns.testcoll_nv', result_no_version='') == '' + # Local collection with MANIFEST.json + - lookup('community.general.collection_version', 'testns.testcoll_mf') == '0.0.1' + # Local collection with no galaxy.yml and no MANIFEST.json + - lookup('community.general.collection_version', 'testns.testcoll_nothing') == '*' + - lookup('community.general.collection_version', 'testns.testcoll_nothing', result_no_version='0.0.0') == '0.0.0' + # Multiple collection names at once + - lookup('community.general.collection_version', 'testns.testcoll', 'testns.testcoll_nv', 'testns.testcoll_nv', 'testns.testcoll_mf', 'foo.bar') + == ['0.0.1', '*', '*', '0.0.1', none] + + - name: Invalid FQCN + set_fact: + test: "{{ query('community.general.collection_version', 'foo.bar.baz') }}" + ignore_errors: true + register: invalid_fqcn + + - name: Validate error message + assert: + that: + - > + '"foo.bar.baz" is not a FQCN' in invalid_fqcn.msg