Implement contructable support for opennebula inventory plugin: keyed… (#4524) (#4549)

* Implement contructable support for opennebula inventory plugin: keyed_groups, compose, groups

* Fixed templating mock issues in unit tests, corrected some linting errors

* trying to make the linter happy

* Now trying to make python2.7 happy

* Added changelog fragment

* changelog fragment needs pluralization

* Update changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml

Co-authored-by: Felix Fontein <felix@fontein.de>

Co-authored-by: Felix Fontein <felix@fontein.de>
(cherry picked from commit 8e72e98adb)

Co-authored-by: Bill Sanders <billysanders@gmail.com>

changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+  - opennebula inventory plugin - complete the implementation of ``constructable`` for opennebula inventory plugin. Now ``keyed_groups``, ``compose``, ``groups`` actually work (https://github.com/ansible-collections/community.general/issues/4497).

plugins/inventory/opennebula.py

@@ -206,28 +206,40 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
     def _populate(self):
         hostname_preference = self.get_option('hostname')
         group_by_labels = self.get_option('group_by_labels')
+        strict = self.get_option('strict')
 
         # Add a top group 'one'
         self.inventory.add_group(group='all')
 
         filter_by_label = self.get_option('filter_by_label')
-        for server in self._retrieve_servers(filter_by_label):
+        servers = self._retrieve_servers(filter_by_label)
+        for server in servers:
+            hostname = server['name']
             # check for labels
             if group_by_labels and server['LABELS']:
                 for label in server['LABELS']:
                     self.inventory.add_group(group=label)
-                    self.inventory.add_host(host=server['name'], group=label)
+                    self.inventory.add_host(host=hostname, group=label)
 
-            self.inventory.add_host(host=server['name'], group='all')
+            self.inventory.add_host(host=hostname, group='all')
 
             for attribute, value in server.items():
-                self.inventory.set_variable(server['name'], attribute, value)
+                self.inventory.set_variable(hostname, attribute, value)
 
             if hostname_preference != 'name':
-                self.inventory.set_variable(server['name'], 'ansible_host', server[hostname_preference])
+                self.inventory.set_variable(hostname, 'ansible_host', server[hostname_preference])
 
             if server.get('SSH_PORT'):
-                self.inventory.set_variable(server['name'], 'ansible_port', server['SSH_PORT'])
+                self.inventory.set_variable(hostname, 'ansible_port', server['SSH_PORT'])
+
+            # handle construcable implementation: get composed variables if any
+            self._set_composite_vars(self.get_option('compose'), server, hostname, strict=strict)
+
+            # groups based on jinja conditionals get added to specific groups
+            self._add_host_to_composed_groups(self.get_option('groups'), server, hostname, strict=strict)
+
+            # groups based on variables associated with them in the inventory
+            self._add_host_to_keyed_groups(self.get_option('keyed_groups'), server, hostname, strict=strict)
 
     def parse(self, inventory, loader, path, cache=True):
         if not HAS_PYONE:

tests/unit/plugins/inventory/fixtures/opennebula_inventory.json

@@ -0,0 +1,222 @@
+[
+    {
+        "DEPLOY_ID": "bcfec9d9-c0d0-4523-b5e7-62993947e94c",
+        "ETIME": 0,
+        "GID": 105,
+        "GNAME": "SW",
+        "HISTORY_RECORDS": {},
+        "ID": 451,
+        "LAST_POLL": 0,
+        "LCM_STATE": 3,
+        "MONITORING": {},
+        "NAME": "terraform_demo_00",
+        "RESCHED": 0,
+        "STATE": 3,
+        "STIME": 1649886492,
+        "TEMPLATE": {
+            "NIC": [
+                {
+                    "AR_ID": "0",
+                    "BRIDGE": "mgmt0",
+                    "BRIDGE_TYPE": "linux",
+                    "CLUSTER_ID": "0",
+                    "IP": "192.168.11.248",
+                    "MAC": "02:00:c0:a8:2b:bb",
+                    "MODEL": "virtio",
+                    "NAME": "NIC0",
+                    "NETWORK": "Infrastructure",
+                    "NETWORK_ID": "0",
+                    "NIC_ID": "0",
+                    "SECURITY_GROUPS": "0,101",
+                    "TARGET": "one-453-0",
+                    "VLAN_ID": "12",
+                    "VN_MAD": "802.1Q"
+                }
+            ],
+            "NIC_DEFAULT": {
+                "MODEL": "virtio"
+            },
+            "TEMPLATE_ID": "28",
+            "TM_MAD_SYSTEM": "shared",
+            "VCPU": "4",
+            "VMID": "453"
+        },
+        "USER_TEMPLATE": {
+            "GUEST_OS": "linux",
+            "INPUTS_ORDER": "",
+            "LABELS": "foo,bench",
+            "LOGO": "images/logos/linux.png",
+            "MEMORY_UNIT_COST": "MB",
+            "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"",
+            "TGROUP": "bench_clients"
+        }
+    },
+    {
+        "DEPLOY_ID": "25895435-5e3a-4d50-a025-e03a7a463abd",
+        "ETIME": 0,
+        "GID": 105,
+        "GNAME": "SW",
+        "HISTORY_RECORDS": {},
+        "ID": 451,
+        "LAST_POLL": 0,
+        "LCM_STATE": 3,
+        "MONITORING": {},
+        "NAME": "terraform_demo_01",
+        "RESCHED": 0,
+        "STATE": 3,
+        "STIME": 1649886492,
+        "TEMPLATE": {
+            "NIC": [
+                {
+                    "AR_ID": "0",
+                    "BRIDGE": "mgmt0",
+                    "BRIDGE_TYPE": "linux",
+                    "CLUSTER_ID": "0",
+                    "IP": "192.168.11.241",
+                    "MAC": "02:00:c0:a8:4b:bb",
+                    "MODEL": "virtio",
+                    "NAME": "NIC0",
+                    "NETWORK": "Infrastructure",
+                    "NETWORK_ID": "0",
+                    "NIC_ID": "0",
+                    "SECURITY_GROUPS": "0,101",
+                    "TARGET": "one-451-0",
+                    "VLAN_ID": "12",
+                    "VN_MAD": "802.1Q"
+                }
+            ],
+            "NIC_DEFAULT": {
+                "MODEL": "virtio"
+            },
+            "TEMPLATE_ID": "28",
+            "TM_MAD_SYSTEM": "shared",
+            "VCPU": "4",
+            "VMID": "451"
+        },
+        "USER_TEMPLATE": {
+            "GUEST_OS": "linux",
+            "INPUTS_ORDER": "",
+            "LABELS": "foo,bench",
+            "LOGO": "images/logos/linux.png",
+            "MEMORY_UNIT_COST": "MB",
+            "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"",
+            "TESTATTR": "testvar",
+            "TGROUP": "bench_clients"
+        }
+    },
+    {
+        "DEPLOY_ID": "2b00c379-3601-45ee-acf5-e7b3ff2b7bca",
+        "ETIME": 0,
+        "GID": 105,
+        "GNAME": "SW",
+        "HISTORY_RECORDS": {},
+        "ID": 451,
+        "LAST_POLL": 0,
+        "LCM_STATE": 3,
+        "MONITORING": {},
+        "NAME": "terraform_demo_srv_00",
+        "RESCHED": 0,
+        "STATE": 3,
+        "STIME": 1649886492,
+        "TEMPLATE": {
+            "NIC": [
+                {
+                    "AR_ID": "0",
+                    "BRIDGE": "mgmt0",
+                    "BRIDGE_TYPE": "linux",
+                    "CLUSTER_ID": "0",
+                    "IP": "192.168.11.247",
+                    "MAC": "02:00:c0:a8:0b:cc",
+                    "MODEL": "virtio",
+                    "NAME": "NIC0",
+                    "NETWORK": "Infrastructure",
+                    "NETWORK_ID": "0",
+                    "NIC_ID": "0",
+                    "SECURITY_GROUPS": "0,101",
+                    "TARGET": "one-452-0",
+                    "VLAN_ID": "12",
+                    "VN_MAD": "802.1Q"
+                }
+            ],
+            "NIC_DEFAULT": {
+                "MODEL": "virtio"
+            },
+            "TEMPLATE_ID": "28",
+            "TM_MAD_SYSTEM": "shared",
+            "VCPU": "4",
+            "VMID": "452"
+        },
+        "USER_TEMPLATE": {
+            "GUEST_OS": "linux",
+            "INPUTS_ORDER": "",
+            "LABELS": "serv,bench",
+            "LOGO": "images/logos/linux.png",
+            "MEMORY_UNIT_COST": "MB",
+            "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"",
+            "TGROUP": "bench_server"
+        }
+    },
+    {
+        "DEPLOY_ID": "97037f55-dd2c-4549-8d24-561a6569e870",
+        "ETIME": 0,
+        "GID": 105,
+        "GNAME": "SW",
+        "HISTORY_RECORDS": {},
+        "ID": 311,
+        "LAST_POLL": 0,
+        "LCM_STATE": 3,
+        "MONITORING": {},
+        "NAME": "bs-windows",
+        "RESCHED": 0,
+        "STATE": 3,
+        "STIME": 1648076254,
+        "TEMPLATE": {
+            "NIC": [
+                {
+                    "AR_ID": "0",
+                    "BRIDGE": "mgmt0",
+                    "BRIDGE_TYPE": "linux",
+                    "CLUSTER_ID": "0",
+                    "IP": "192.168.11.209",
+                    "MAC": "02:00:c0:a8:0b:dd",
+                    "MODEL": "virtio",
+                    "NAME": "NIC0",
+                    "NETWORK": "Infrastructure",
+                    "NETWORK_ID": "0",
+                    "NETWORK_UNAME": "admin",
+                    "NIC_ID": "0",
+                    "SECURITY_GROUPS": "0,101",
+                    "TARGET": "one-311-0",
+                    "VLAN_ID": "12",
+                    "VN_MAD": "802.1Q"
+                },
+                [
+                    "TEMPLATE_ID",
+                    "23"
+                ],
+                [
+                    "TM_MAD_SYSTEM",
+                    "shared"
+                ],
+                [
+                    "VCPU",
+                    "4"
+                ],
+                [
+                    "VMID",
+                    "311"
+                ]
+            ]
+        },
+        "UID": 22,
+        "UNAME": "bsanders",
+        "USER_TEMPLATE": {
+            "GUEST_OS": "windows",
+            "INPUTS_ORDER": "",
+            "LABELS": "serv",
+            "HYPERVISOR": "kvm",
+            "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"",
+            "SET_HOSTNAME": "windows"
+        }
+    }
+]

tests/unit/plugins/inventory/test_opennebula.py

@@ -9,14 +9,18 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from collections import OrderedDict
+import json
 
 import pytest
 
 from ansible.inventory.data import InventoryData
+from ansible.parsing.dataloader import DataLoader
+from ansible.template import Templar
 from ansible_collections.community.general.plugins.inventory.opennebula import InventoryModule
+from ansible_collections.community.general.tests.unit.compat.mock import create_autospec
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def inventory():
     r = InventoryModule()
     r.inventory = InventoryData()
@@ -33,6 +37,18 @@ def test_verify_file_bad_config(inventory):
     assert inventory.verify_file('foobar.opennebula.yml') is False
 
 
+def get_vm_pool_json():
+    with open('tests/unit/plugins/inventory/fixtures/opennebula_inventory.json', 'r') as json_file:
+        jsondata = json.load(json_file)
+
+    data = type('pyone.bindings.VM_POOLSub', (object,), {'VM': []})()
+
+    for fake_server in jsondata:
+        data.VM.append(type('pyone.bindings.VMType90Sub', (object,), fake_server)())
+
+    return data
+
+
 def get_vm_pool():
     data = type('pyone.bindings.VM_POOLSub', (object,), {'VM': []})()
 
@@ -195,36 +211,99 @@ def get_vm_pool():
     return data
 
 
-def get_option(option):
+options_base_test = {
-    if option == 'api_url':
+    'api_url': 'https://opennebula:2633/RPC2',
-        return 'https://opennebula:2633/RPC2'
+    'api_username': 'username',
-    if option == 'api_username':
+    'api_password': 'password',
-        return 'username'
+    'api_authfile': '~/.one/one_auth',
-    elif option == 'api_password':
+    'hostname': 'v4_first_ip',
-        return 'password'
+    'group_by_labels': True,
-    elif option == 'api_authfile':
+    'filter_by_label': None,
-        return '~/.one/one_auth'
+}
-    elif option == 'hostname':
+
-        return 'v4_first_ip'
+options_constructable_test = options_base_test.copy()
-    elif option == 'group_by_labels':
+options_constructable_test.update({
-        return True
+    'compose': {'is_linux': "GUEST_OS == 'linux'"},
-    elif option == 'filter_by_label':
+    'filter_by_label': 'bench',
-        return None
+    'groups': {
-    else:
+        'benchmark_clients': "TGROUP.endswith('clients')",
-        return False
+        'lin': 'is_linux == True'
+    },
+    'keyed_groups': [{'key': 'TGROUP', 'prefix': 'tgroup'}],
+
+})
+
+
+# given a dictionary `opts_dict`, return a function that behaves like ansible's inventory get_options
+def mk_get_options(opts_dict):
+    def inner(opt):
+        return opts_dict.get(opt, False)
+    return inner
 
 
 def test_get_connection_info(inventory, mocker):
-    inventory.get_option = mocker.MagicMock(side_effect=get_option)
+    inventory.get_option = mocker.MagicMock(side_effect=mk_get_options(options_base_test))
 
     auth = inventory._get_connection_info()
     assert (auth.username and auth.password)
 
 
+def test_populate_constructable_templating(inventory, mocker):
+    # bypass API fetch call
+    inventory._get_vm_pool = mocker.MagicMock(side_effect=get_vm_pool_json)
+    inventory.get_option = mocker.MagicMock(side_effect=mk_get_options(options_constructable_test))
+
+    # the templating engine is needed for the constructable groups/vars
+    # so give that some fake data and instantiate it.
+    fake_config_filepath = '/fake/opennebula.yml'
+    fake_cache = {fake_config_filepath: options_constructable_test.copy()}
+    fake_cache[fake_config_filepath]['plugin'] = 'community.general.opennebula'
+    dataloader = create_autospec(DataLoader, instance=True)
+    dataloader._FILE_CACHE = fake_cache
+    inventory.templar = Templar(loader=dataloader)
+
+    inventory._populate()
+
+    # note the vm_pool (and json data file) has four hosts,
+    # but options_constructable_test asks ansible to filter it out
+    assert len(get_vm_pool_json().VM) == 4
+    assert set([vm.NAME for vm in get_vm_pool_json().VM]) == set([
+        'terraform_demo_00',
+        'terraform_demo_01',
+        'terraform_demo_srv_00',
+        'bs-windows',
+    ])
+    assert set(inventory.inventory.hosts) == set(['terraform_demo_00', 'terraform_demo_01', 'terraform_demo_srv_00'])
+
+    host_demo00 = inventory.inventory.get_host('terraform_demo_00')
+    host_demo01 = inventory.inventory.get_host('terraform_demo_01')
+    host_demosrv = inventory.inventory.get_host('terraform_demo_srv_00')
+
+    assert 'benchmark_clients' in inventory.inventory.groups
+    assert 'lin' in inventory.inventory.groups
+    assert inventory.inventory.groups['benchmark_clients'].hosts == [host_demo00, host_demo01]
+    assert inventory.inventory.groups['lin'].hosts == [host_demo00, host_demo01, host_demosrv]
+
+    # test group by label:
+    assert 'bench' in inventory.inventory.groups
+    assert 'foo' in inventory.inventory.groups
+    assert inventory.inventory.groups['bench'].hosts == [host_demo00, host_demo01, host_demosrv]
+    assert inventory.inventory.groups['serv'].hosts == [host_demosrv]
+    assert inventory.inventory.groups['foo'].hosts == [host_demo00, host_demo01]
+
+    # test `compose` transforms GUEST_OS=Linux to is_linux == True
+    assert host_demo00.get_vars()['GUEST_OS'] == 'linux'
+    assert host_demo00.get_vars()['is_linux'] is True
+
+    # test `keyed_groups`
+    assert inventory.inventory.groups['tgroup_bench_clients'].hosts == [host_demo00, host_demo01]
+    assert inventory.inventory.groups['tgroup_bench_server'].hosts == [host_demosrv]
+
+
 def test_populate(inventory, mocker):
     # bypass API fetch call
     inventory._get_vm_pool = mocker.MagicMock(side_effect=get_vm_pool)
-    inventory.get_option = mocker.MagicMock(side_effect=get_option)
+    inventory.get_option = mocker.MagicMock(side_effect=mk_get_options(options_base_test))
     inventory._populate()
 
     # get different hosts