Add collection_version lookup (#3633)

* Add collection_verison lookup plugin.

* Document ansible-base 2.10 and ansible-core 2.11 special case.

* Add default for result_not_found.

* Simplify not existing test.

* Move metadata loading to function.

* Update plugins/lookup/collection_version.py

Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com>

* Extend tests.

* Fix tests for ansible-core < 2.12.

Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com>

.github/BOTMETA.yml

@@ -175,6 +175,8 @@ files:
     labels: lookups
   $lookups/cartesian.py: {}
   $lookups/chef_databag.py: {}
+  $lookups/collection_version.py:
+    maintainers: felixfontein
   $lookups/consul_kv.py: {}
   $lookups/credstash.py: {}
   $lookups/cyberarkpassword.py:

plugins/lookup/collection_version.py

@@ -0,0 +1,138 @@
+# (c) 2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = """
+name: collection_version
+author: Felix Fontein (@felixfontein)
+version_added: "4.0.0"
+short_description: Retrieves the version of an installed collection
+description:
+  - This lookup allows to query the version of an installed collection, and to determine whether a
+    collection is installed at all.
+  - By default it returns C(none) for non-existing collections and C(*) for collections without a
+    version number. The latter should only happen in development environments, or when installing
+    a collection from git which has no version in its C(galaxy.yml). This behavior can be adjusted
+    by providing other values with I(result_not_found) and I(result_no_version).
+options:
+  _terms:
+    description:
+      - The collections to look for.
+      - For example C(community.general).
+    type: list
+    elements: str
+    required: true
+  result_not_found:
+    description:
+      - The value to return when the collection could not be found.
+      - By default, C(none) is returned.
+    type: string
+    default: ~
+  result_no_version:
+    description:
+      - The value to return when the collection has no version number.
+      - This can happen for collections installed from git which do not have a version number
+        in C(galaxy.yml).
+      - By default, C(*) is returned.
+    type: string
+    default: '*'
+"""
+
+EXAMPLES = """
+- name: Check version of community.general
+  ansible.builtin.debug:
+    msg: "community.general version {{ lookup('community.general.collection_version', 'community.general') }}"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - The version number of the collections listed as input.
+      - If a collection can not be found, it will return the value provided in I(result_not_found).
+        By default, this is C(none).
+      - If a collection can be found, but the version not identified, it will return the value provided in
+        I(result_no_version). By default, this is C(*). This can happen for collections installed
+        from git which do not have a version number in C(galaxy.yml).
+    type: list
+    elements: str
+"""
+
+import json
+import os
+import re
+
+import yaml
+
+from ansible.errors import AnsibleLookupError
+from ansible.module_utils.compat.importlib import import_module
+from ansible.plugins.lookup import LookupBase
+
+
+FQCN_RE = re.compile(r'^[A-Za-z0-9_]+\.[A-Za-z0-9_]+$')
+
+
+def load_collection_meta_manifest(manifest_path):
+    with open(manifest_path, 'rb') as f:
+        meta = json.load(f)
+    return {
+        'version': meta['collection_info']['version'],
+    }
+
+
+def load_collection_meta_galaxy(galaxy_path, no_version='*'):
+    with open(galaxy_path, 'rb') as f:
+        meta = yaml.safe_load(f)
+    return {
+        'version': meta.get('version') or no_version,
+    }
+
+
+def load_collection_meta(collection_pkg, no_version='*'):
+    path = os.path.dirname(collection_pkg.__file__)
+
+    # Try to load MANIFEST.json
+    manifest_path = os.path.join(path, 'MANIFEST.json')
+    if os.path.exists(manifest_path):
+        return load_collection_meta_manifest(manifest_path)
+
+    # Try to load galaxy.y(a)ml
+    galaxy_path = os.path.join(path, 'galaxy.yml')
+    galaxy_alt_path = os.path.join(path, 'galaxy.yaml')
+    # galaxy.yaml was only supported in ansible-base 2.10 and ansible-core 2.11. Support was removed
+    # in https://github.com/ansible/ansible/commit/595413d11346b6f26bb3d9df2d8e05f2747508a3 for
+    # ansible-core 2.12.
+    for path in (galaxy_path, galaxy_alt_path):
+        if os.path.exists(path):
+            return load_collection_meta_galaxy(path, no_version=no_version)
+
+    return {}
+
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables=None, **kwargs):
+        result = []
+        self.set_options(var_options=variables, direct=kwargs)
+        not_found = self.get_option('result_not_found')
+        no_version = self.get_option('result_no_version')
+
+        for term in terms:
+            if not FQCN_RE.match(term):
+                raise AnsibleLookupError('"{term}" is not a FQCN'.format(term=term))
+
+            try:
+                collection_pkg = import_module('ansible_collections.{fqcn}'.format(fqcn=term))
+            except ImportError:
+                # Collection not found
+                result.append(not_found)
+                continue
+
+            try:
+                data = load_collection_meta(collection_pkg, no_version=no_version)
+            except Exception as exc:
+                raise AnsibleLookupError('Error while loading metadata for {fqcn}: {error}'.format(fqcn=term, error=exc))
+
+            result.append(data.get('version', no_version))
+
+        return result

tests/integration/targets/lookup_collection_version/aliases

@@ -0,0 +1 @@
+shippable/posix/group3

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/galaxy.yml

@@ -0,0 +1,7 @@
+namespace: testns
+name: testcoll
+version: 0.0.1
+authors:
+  - Ansible (https://github.com/ansible)
+description: null
+tags: [community]

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll/plugins/modules/collection_module.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: collection_module
+short_description: Test collection module
+description:
+  - This is a test module in a local collection.
+author: "Felix Fontein (@felixfontein)"
+options: {}
+'''
+
+EXAMPLES = ''' # '''
+
+RETURN = ''' # '''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    AnsibleModule(argument_spec={}).exit_json()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/FILES.json

@@ -0,0 +1,40 @@
+{
+ "files": [
+  {
+   "name": ".",
+   "ftype": "dir",
+   "chksum_type": null,
+   "chksum_sha256": null,
+   "format": 1
+  },
+  {
+   "name": "README.md",
+   "ftype": "file",
+   "chksum_type": "sha256",
+   "chksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+   "format": 1
+  },
+  {
+   "name": "plugins",
+   "ftype": "dir",
+   "chksum_type": null,
+   "chksum_sha256": null,
+   "format": 1
+  },
+  {
+   "name": "plugins/modules",
+   "ftype": "dir",
+   "chksum_type": null,
+   "chksum_sha256": null,
+   "format": 1
+  },
+  {
+   "name": "plugins/modules/collection_module.py",
+   "ftype": "file",
+   "chksum_type": "sha256",
+   "chksum_sha256": "c3f0114d080c409c58c8846be8da7b91137b38eaf2d24f72a4a61a303f925f4d",
+   "format": 1
+  }
+ ],
+ "format": 1
+}

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/MANIFEST.json

@@ -0,0 +1,30 @@
+{
+ "collection_info": {
+  "namespace": "testns",
+  "name": "testcoll_mf",
+  "version": "0.0.1",
+  "authors": [
+   "Ansible (https://github.com/ansible)"
+  ],
+  "readme": "README.md",
+  "tags": [
+   "community"
+  ],
+  "description": null,
+  "license": [],
+  "license_file": null,
+  "dependencies": {},
+  "repository": null,
+  "documentation": null,
+  "homepage": null,
+  "issues": null
+ },
+ "file_manifest_file": {
+  "name": "FILES.json",
+  "ftype": "file",
+  "chksum_type": "sha256",
+  "chksum_sha256": "025818f18fcae5c9f78d778ae6e246ecffed6d56a886ffbc145cb66d54e9951e",
+  "format": 1
+ },
+ "format": 1
+}

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_mf/plugins/modules/collection_module.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: collection_module
+short_description: Test collection module
+description:
+  - This is a test module in a local collection.
+author: "Felix Fontein (@felixfontein)"
+options: {}
+'''
+
+EXAMPLES = ''' # '''
+
+RETURN = ''' # '''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    AnsibleModule(argument_spec={}).exit_json()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nothing/plugins/modules/collection_module.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: collection_module
+short_description: Test collection module
+description:
+  - This is a test module in a local collection.
+author: "Felix Fontein (@felixfontein)"
+options: {}
+'''
+
+EXAMPLES = ''' # '''
+
+RETURN = ''' # '''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    AnsibleModule(argument_spec={}).exit_json()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/galaxy.yml

@@ -0,0 +1,6 @@
+namespace: testns
+name: testcoll_nv
+authors:
+  - Ansible (https://github.com/ansible)
+description: null
+tags: [community]

tests/integration/targets/lookup_collection_version/collections/ansible_collections/testns/testcoll_nv/plugins/modules/collection_module.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: collection_module
+short_description: Test collection module
+description:
+  - This is a test module in a local collection.
+author: "Felix Fontein (@felixfontein)"
+options: {}
+'''
+
+EXAMPLES = ''' # '''
+
+RETURN = ''' # '''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    AnsibleModule(argument_spec={}).exit_json()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/lookup_collection_version/library/local_module.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2021, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: local_module
+short_description: Test local module
+description:
+  - This is a test module locally next to a playbook.
+author: "Felix Fontein (@felixfontein)"
+options: {}
+'''
+
+EXAMPLES = ''' # '''
+
+RETURN = ''' # '''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    AnsibleModule(argument_spec={}).exit_json()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/lookup_collection_version/runme.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export ANSIBLE_TEST_PREFER_VENV=1  # see https://github.com/ansible/ansible/pull/73000#issuecomment-757012395; can be removed once Ansible 2.9 and ansible-base 2.10 support has been dropped
+source virtualenv.sh
+
+# The collection loader ignores paths which have more than one ansible_collections in it.
+# That's why we have to copy this directory to a temporary place and run the test there.
+
+# Create temporary folder
+TEMPDIR=$(mktemp -d)
+trap '{ rm -rf ${TEMPDIR}; }' EXIT
+
+cp -r . "${TEMPDIR}"
+cd "${TEMPDIR}"
+
+ansible-playbook runme.yml "$@"

tests/integration/targets/lookup_collection_version/runme.yml

@@ -0,0 +1,35 @@
+- hosts: localhost
+  tasks:
+    - name: Test collection_version
+      assert:
+        that:
+          # Collection that does not exist
+          - query('community.general.collection_version', 'foo.bar') == [none]
+          - lookup('community.general.collection_version', 'foo.bar', result_not_found='foo') == 'foo'
+          # Collection that exists
+          - lookup('community.general.collection_version', 'community.general') is string
+          # Local collection
+          - lookup('community.general.collection_version', 'testns.testcoll') == '0.0.1'
+          # Local collection with no version
+          - lookup('community.general.collection_version', 'testns.testcoll_nv') == '*'
+          - lookup('community.general.collection_version', 'testns.testcoll_nv', result_no_version='') == ''
+          # Local collection with MANIFEST.json
+          - lookup('community.general.collection_version', 'testns.testcoll_mf') == '0.0.1'
+          # Local collection with no galaxy.yml and no MANIFEST.json
+          - lookup('community.general.collection_version', 'testns.testcoll_nothing') == '*'
+          - lookup('community.general.collection_version', 'testns.testcoll_nothing', result_no_version='0.0.0') == '0.0.0'
+          # Multiple collection names at once
+          - lookup('community.general.collection_version', 'testns.testcoll', 'testns.testcoll_nv', 'testns.testcoll_nv', 'testns.testcoll_mf', 'foo.bar')
+            == ['0.0.1', '*', '*', '0.0.1', none]
+
+    - name: Invalid FQCN
+      set_fact:
+        test: "{{ query('community.general.collection_version', 'foo.bar.baz') }}"
+      ignore_errors: true
+      register: invalid_fqcn
+
+    - name: Validate error message
+      assert:
+        that:
+          - >
+            '"foo.bar.baz" is not a FQCN' in invalid_fqcn.msg