From 8e72e98adbf9ad0b16523605ca6c1d0deff170c6 Mon Sep 17 00:00:00 2001 From: Bill Sanders Date: Thu, 21 Apr 2022 04:16:15 -0700 Subject: [PATCH] =?UTF-8?q?Implement=20contructable=20support=20for=20open?= =?UTF-8?q?nebula=20inventory=20plugin:=20keyed=E2=80=A6=20(#4524)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement contructable support for opennebula inventory plugin: keyed_groups, compose, groups * Fixed templating mock issues in unit tests, corrected some linting errors * trying to make the linter happy * Now trying to make python2.7 happy * Added changelog fragment * changelog fragment needs pluralization * Update changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml Co-authored-by: Felix Fontein Co-authored-by: Felix Fontein --- ...ventory-plugin-to-match-documentation.yaml | 3 + plugins/inventory/opennebula.py | 24 +- .../fixtures/opennebula_inventory.json | 222 ++++++++++++++++++ .../unit/plugins/inventory/test_opennebula.py | 119 ++++++++-- 4 files changed, 342 insertions(+), 26 deletions(-) create mode 100644 changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml create mode 100644 tests/unit/plugins/inventory/fixtures/opennebula_inventory.json diff --git a/changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml b/changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml new file mode 100644 index 0000000000..e5b32c181b --- /dev/null +++ b/changelogs/fragments/4524-update-opennebula-inventory-plugin-to-match-documentation.yaml @@ -0,0 +1,3 @@ +--- +bugfixes: + - opennebula inventory plugin - complete the implementation of ``constructable`` for opennebula inventory plugin. Now ``keyed_groups``, ``compose``, ``groups`` actually work (https://github.com/ansible-collections/community.general/issues/4497). diff --git a/plugins/inventory/opennebula.py b/plugins/inventory/opennebula.py index d967e13f7a..7822240627 100644 --- a/plugins/inventory/opennebula.py +++ b/plugins/inventory/opennebula.py @@ -206,28 +206,40 @@ class InventoryModule(BaseInventoryPlugin, Constructable): def _populate(self): hostname_preference = self.get_option('hostname') group_by_labels = self.get_option('group_by_labels') + strict = self.get_option('strict') # Add a top group 'one' self.inventory.add_group(group='all') filter_by_label = self.get_option('filter_by_label') - for server in self._retrieve_servers(filter_by_label): + servers = self._retrieve_servers(filter_by_label) + for server in servers: + hostname = server['name'] # check for labels if group_by_labels and server['LABELS']: for label in server['LABELS']: self.inventory.add_group(group=label) - self.inventory.add_host(host=server['name'], group=label) + self.inventory.add_host(host=hostname, group=label) - self.inventory.add_host(host=server['name'], group='all') + self.inventory.add_host(host=hostname, group='all') for attribute, value in server.items(): - self.inventory.set_variable(server['name'], attribute, value) + self.inventory.set_variable(hostname, attribute, value) if hostname_preference != 'name': - self.inventory.set_variable(server['name'], 'ansible_host', server[hostname_preference]) + self.inventory.set_variable(hostname, 'ansible_host', server[hostname_preference]) if server.get('SSH_PORT'): - self.inventory.set_variable(server['name'], 'ansible_port', server['SSH_PORT']) + self.inventory.set_variable(hostname, 'ansible_port', server['SSH_PORT']) + + # handle construcable implementation: get composed variables if any + self._set_composite_vars(self.get_option('compose'), server, hostname, strict=strict) + + # groups based on jinja conditionals get added to specific groups + self._add_host_to_composed_groups(self.get_option('groups'), server, hostname, strict=strict) + + # groups based on variables associated with them in the inventory + self._add_host_to_keyed_groups(self.get_option('keyed_groups'), server, hostname, strict=strict) def parse(self, inventory, loader, path, cache=True): if not HAS_PYONE: diff --git a/tests/unit/plugins/inventory/fixtures/opennebula_inventory.json b/tests/unit/plugins/inventory/fixtures/opennebula_inventory.json new file mode 100644 index 0000000000..f7be74f90b --- /dev/null +++ b/tests/unit/plugins/inventory/fixtures/opennebula_inventory.json @@ -0,0 +1,222 @@ +[ + { + "DEPLOY_ID": "bcfec9d9-c0d0-4523-b5e7-62993947e94c", + "ETIME": 0, + "GID": 105, + "GNAME": "SW", + "HISTORY_RECORDS": {}, + "ID": 451, + "LAST_POLL": 0, + "LCM_STATE": 3, + "MONITORING": {}, + "NAME": "terraform_demo_00", + "RESCHED": 0, + "STATE": 3, + "STIME": 1649886492, + "TEMPLATE": { + "NIC": [ + { + "AR_ID": "0", + "BRIDGE": "mgmt0", + "BRIDGE_TYPE": "linux", + "CLUSTER_ID": "0", + "IP": "192.168.11.248", + "MAC": "02:00:c0:a8:2b:bb", + "MODEL": "virtio", + "NAME": "NIC0", + "NETWORK": "Infrastructure", + "NETWORK_ID": "0", + "NIC_ID": "0", + "SECURITY_GROUPS": "0,101", + "TARGET": "one-453-0", + "VLAN_ID": "12", + "VN_MAD": "802.1Q" + } + ], + "NIC_DEFAULT": { + "MODEL": "virtio" + }, + "TEMPLATE_ID": "28", + "TM_MAD_SYSTEM": "shared", + "VCPU": "4", + "VMID": "453" + }, + "USER_TEMPLATE": { + "GUEST_OS": "linux", + "INPUTS_ORDER": "", + "LABELS": "foo,bench", + "LOGO": "images/logos/linux.png", + "MEMORY_UNIT_COST": "MB", + "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"", + "TGROUP": "bench_clients" + } + }, + { + "DEPLOY_ID": "25895435-5e3a-4d50-a025-e03a7a463abd", + "ETIME": 0, + "GID": 105, + "GNAME": "SW", + "HISTORY_RECORDS": {}, + "ID": 451, + "LAST_POLL": 0, + "LCM_STATE": 3, + "MONITORING": {}, + "NAME": "terraform_demo_01", + "RESCHED": 0, + "STATE": 3, + "STIME": 1649886492, + "TEMPLATE": { + "NIC": [ + { + "AR_ID": "0", + "BRIDGE": "mgmt0", + "BRIDGE_TYPE": "linux", + "CLUSTER_ID": "0", + "IP": "192.168.11.241", + "MAC": "02:00:c0:a8:4b:bb", + "MODEL": "virtio", + "NAME": "NIC0", + "NETWORK": "Infrastructure", + "NETWORK_ID": "0", + "NIC_ID": "0", + "SECURITY_GROUPS": "0,101", + "TARGET": "one-451-0", + "VLAN_ID": "12", + "VN_MAD": "802.1Q" + } + ], + "NIC_DEFAULT": { + "MODEL": "virtio" + }, + "TEMPLATE_ID": "28", + "TM_MAD_SYSTEM": "shared", + "VCPU": "4", + "VMID": "451" + }, + "USER_TEMPLATE": { + "GUEST_OS": "linux", + "INPUTS_ORDER": "", + "LABELS": "foo,bench", + "LOGO": "images/logos/linux.png", + "MEMORY_UNIT_COST": "MB", + "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"", + "TESTATTR": "testvar", + "TGROUP": "bench_clients" + } + }, + { + "DEPLOY_ID": "2b00c379-3601-45ee-acf5-e7b3ff2b7bca", + "ETIME": 0, + "GID": 105, + "GNAME": "SW", + "HISTORY_RECORDS": {}, + "ID": 451, + "LAST_POLL": 0, + "LCM_STATE": 3, + "MONITORING": {}, + "NAME": "terraform_demo_srv_00", + "RESCHED": 0, + "STATE": 3, + "STIME": 1649886492, + "TEMPLATE": { + "NIC": [ + { + "AR_ID": "0", + "BRIDGE": "mgmt0", + "BRIDGE_TYPE": "linux", + "CLUSTER_ID": "0", + "IP": "192.168.11.247", + "MAC": "02:00:c0:a8:0b:cc", + "MODEL": "virtio", + "NAME": "NIC0", + "NETWORK": "Infrastructure", + "NETWORK_ID": "0", + "NIC_ID": "0", + "SECURITY_GROUPS": "0,101", + "TARGET": "one-452-0", + "VLAN_ID": "12", + "VN_MAD": "802.1Q" + } + ], + "NIC_DEFAULT": { + "MODEL": "virtio" + }, + "TEMPLATE_ID": "28", + "TM_MAD_SYSTEM": "shared", + "VCPU": "4", + "VMID": "452" + }, + "USER_TEMPLATE": { + "GUEST_OS": "linux", + "INPUTS_ORDER": "", + "LABELS": "serv,bench", + "LOGO": "images/logos/linux.png", + "MEMORY_UNIT_COST": "MB", + "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"", + "TGROUP": "bench_server" + } + }, + { + "DEPLOY_ID": "97037f55-dd2c-4549-8d24-561a6569e870", + "ETIME": 0, + "GID": 105, + "GNAME": "SW", + "HISTORY_RECORDS": {}, + "ID": 311, + "LAST_POLL": 0, + "LCM_STATE": 3, + "MONITORING": {}, + "NAME": "bs-windows", + "RESCHED": 0, + "STATE": 3, + "STIME": 1648076254, + "TEMPLATE": { + "NIC": [ + { + "AR_ID": "0", + "BRIDGE": "mgmt0", + "BRIDGE_TYPE": "linux", + "CLUSTER_ID": "0", + "IP": "192.168.11.209", + "MAC": "02:00:c0:a8:0b:dd", + "MODEL": "virtio", + "NAME": "NIC0", + "NETWORK": "Infrastructure", + "NETWORK_ID": "0", + "NETWORK_UNAME": "admin", + "NIC_ID": "0", + "SECURITY_GROUPS": "0,101", + "TARGET": "one-311-0", + "VLAN_ID": "12", + "VN_MAD": "802.1Q" + }, + [ + "TEMPLATE_ID", + "23" + ], + [ + "TM_MAD_SYSTEM", + "shared" + ], + [ + "VCPU", + "4" + ], + [ + "VMID", + "311" + ] + ] + }, + "UID": 22, + "UNAME": "bsanders", + "USER_TEMPLATE": { + "GUEST_OS": "windows", + "INPUTS_ORDER": "", + "LABELS": "serv", + "HYPERVISOR": "kvm", + "SCHED_REQUIREMENTS": "ARCH=\"x86_64\"", + "SET_HOSTNAME": "windows" + } + } +] diff --git a/tests/unit/plugins/inventory/test_opennebula.py b/tests/unit/plugins/inventory/test_opennebula.py index ee450d2ef5..88a5f29d2d 100644 --- a/tests/unit/plugins/inventory/test_opennebula.py +++ b/tests/unit/plugins/inventory/test_opennebula.py @@ -9,14 +9,18 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from collections import OrderedDict +import json import pytest from ansible.inventory.data import InventoryData +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar from ansible_collections.community.general.plugins.inventory.opennebula import InventoryModule +from ansible_collections.community.general.tests.unit.compat.mock import create_autospec -@pytest.fixture(scope="module") +@pytest.fixture def inventory(): r = InventoryModule() r.inventory = InventoryData() @@ -33,6 +37,18 @@ def test_verify_file_bad_config(inventory): assert inventory.verify_file('foobar.opennebula.yml') is False +def get_vm_pool_json(): + with open('tests/unit/plugins/inventory/fixtures/opennebula_inventory.json', 'r') as json_file: + jsondata = json.load(json_file) + + data = type('pyone.bindings.VM_POOLSub', (object,), {'VM': []})() + + for fake_server in jsondata: + data.VM.append(type('pyone.bindings.VMType90Sub', (object,), fake_server)()) + + return data + + def get_vm_pool(): data = type('pyone.bindings.VM_POOLSub', (object,), {'VM': []})() @@ -195,36 +211,99 @@ def get_vm_pool(): return data -def get_option(option): - if option == 'api_url': - return 'https://opennebula:2633/RPC2' - if option == 'api_username': - return 'username' - elif option == 'api_password': - return 'password' - elif option == 'api_authfile': - return '~/.one/one_auth' - elif option == 'hostname': - return 'v4_first_ip' - elif option == 'group_by_labels': - return True - elif option == 'filter_by_label': - return None - else: - return False +options_base_test = { + 'api_url': 'https://opennebula:2633/RPC2', + 'api_username': 'username', + 'api_password': 'password', + 'api_authfile': '~/.one/one_auth', + 'hostname': 'v4_first_ip', + 'group_by_labels': True, + 'filter_by_label': None, +} + +options_constructable_test = options_base_test.copy() +options_constructable_test.update({ + 'compose': {'is_linux': "GUEST_OS == 'linux'"}, + 'filter_by_label': 'bench', + 'groups': { + 'benchmark_clients': "TGROUP.endswith('clients')", + 'lin': 'is_linux == True' + }, + 'keyed_groups': [{'key': 'TGROUP', 'prefix': 'tgroup'}], + +}) + + +# given a dictionary `opts_dict`, return a function that behaves like ansible's inventory get_options +def mk_get_options(opts_dict): + def inner(opt): + return opts_dict.get(opt, False) + return inner def test_get_connection_info(inventory, mocker): - inventory.get_option = mocker.MagicMock(side_effect=get_option) + inventory.get_option = mocker.MagicMock(side_effect=mk_get_options(options_base_test)) auth = inventory._get_connection_info() assert (auth.username and auth.password) +def test_populate_constructable_templating(inventory, mocker): + # bypass API fetch call + inventory._get_vm_pool = mocker.MagicMock(side_effect=get_vm_pool_json) + inventory.get_option = mocker.MagicMock(side_effect=mk_get_options(options_constructable_test)) + + # the templating engine is needed for the constructable groups/vars + # so give that some fake data and instantiate it. + fake_config_filepath = '/fake/opennebula.yml' + fake_cache = {fake_config_filepath: options_constructable_test.copy()} + fake_cache[fake_config_filepath]['plugin'] = 'community.general.opennebula' + dataloader = create_autospec(DataLoader, instance=True) + dataloader._FILE_CACHE = fake_cache + inventory.templar = Templar(loader=dataloader) + + inventory._populate() + + # note the vm_pool (and json data file) has four hosts, + # but options_constructable_test asks ansible to filter it out + assert len(get_vm_pool_json().VM) == 4 + assert set([vm.NAME for vm in get_vm_pool_json().VM]) == set([ + 'terraform_demo_00', + 'terraform_demo_01', + 'terraform_demo_srv_00', + 'bs-windows', + ]) + assert set(inventory.inventory.hosts) == set(['terraform_demo_00', 'terraform_demo_01', 'terraform_demo_srv_00']) + + host_demo00 = inventory.inventory.get_host('terraform_demo_00') + host_demo01 = inventory.inventory.get_host('terraform_demo_01') + host_demosrv = inventory.inventory.get_host('terraform_demo_srv_00') + + assert 'benchmark_clients' in inventory.inventory.groups + assert 'lin' in inventory.inventory.groups + assert inventory.inventory.groups['benchmark_clients'].hosts == [host_demo00, host_demo01] + assert inventory.inventory.groups['lin'].hosts == [host_demo00, host_demo01, host_demosrv] + + # test group by label: + assert 'bench' in inventory.inventory.groups + assert 'foo' in inventory.inventory.groups + assert inventory.inventory.groups['bench'].hosts == [host_demo00, host_demo01, host_demosrv] + assert inventory.inventory.groups['serv'].hosts == [host_demosrv] + assert inventory.inventory.groups['foo'].hosts == [host_demo00, host_demo01] + + # test `compose` transforms GUEST_OS=Linux to is_linux == True + assert host_demo00.get_vars()['GUEST_OS'] == 'linux' + assert host_demo00.get_vars()['is_linux'] is True + + # test `keyed_groups` + assert inventory.inventory.groups['tgroup_bench_clients'].hosts == [host_demo00, host_demo01] + assert inventory.inventory.groups['tgroup_bench_server'].hosts == [host_demosrv] + + def test_populate(inventory, mocker): # bypass API fetch call inventory._get_vm_pool = mocker.MagicMock(side_effect=get_vm_pool) - inventory.get_option = mocker.MagicMock(side_effect=get_option) + inventory.get_option = mocker.MagicMock(side_effect=mk_get_options(options_base_test)) inventory._populate() # get different hosts