Added OpenNebula inventory plugin (#810)

* Added OpenNebula inventory plugin

Signed-off-by: Kristián Feldsam <feldsam@gmail.com>

* Apply suggestions from code review

Co-authored-by: Andrew Klychkov <aaklychkov@mail.ru>

* Apply suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>

* Removed matching inventory yaml files ending with "one"

Too general word

* Apply suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>

* Apply suggestions from code review

Co-authored-by: Alexei Znamensky <103110+russoz@users.noreply.github.com>
Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Added BOTMETA

Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Moved import

Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Fix indentation problem

Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Added group_by_labels, refactored so can be unit tested

Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Added unit tests

Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Removed blank line

Signed-off-by: Kristian Feldsam <feldsam@gmail.com>

* Apply suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>

Co-authored-by: Andrew Klychkov <aaklychkov@mail.ru>
Co-authored-by: Felix Fontein <felix@fontein.de>
Co-authored-by: Alexei Znamensky <103110+russoz@users.noreply.github.com>

.github/BOTMETA.yml

@@ -153,6 +153,10 @@ files:
   $inventories/nmap.py: {}
   $inventories/online.py:
     maintainers: sieben
+  $inventories/opennebula.py:
+    maintainers: feldsam
+    labels: cloud opennebula
+    keywords: opennebula dynamic inventory script
   $inventories/proxmox.py:
     maintainers: $team_virt ilijamt
   $inventories/icinga2.py:

plugins/inventory/opennebula.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, FELDSAM s.r.o. - FeldHost™ <support@feldhost.cz>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+    name: opennebula
+    author:
+        - Kristian Feldsam (@feldsam)
+    short_description: OpenNebula inventory source
+    version_added: "3.8.0"
+    extends_documentation_fragment:
+        - constructed
+    description:
+        - Get inventory hosts from OpenNebula cloud.
+        - Uses an YAML configuration file ending with either I(opennebula.yml) or I(opennebula.yaml)
+          to set parameter values.
+        - Uses I(api_authfile), C(~/.one/one_auth), or C(ONE_AUTH) pointing to a OpenNebula credentials file.
+    options:
+        plugin:
+            description: Token that ensures this is a source file for the 'opennebula' plugin.
+            type: string
+            required: true
+            choices: [ community.general.opennebula ]
+        api_url:
+            description:
+              - URL of the OpenNebula RPC server.
+              - It is recommended to use HTTPS so that the username/password are not
+                transferred over the network unencrypted.
+              - If not set then the value of the C(ONE_URL) environment variable is used.
+            env:
+              - name: ONE_URL
+            required: True
+            type: string
+        api_username:
+            description:
+              - Name of the user to login into the OpenNebula RPC server. If not set
+                then the value of the C(ONE_USERNAME) environment variable is used.
+            env:
+              - name: ONE_USERNAME
+            type: string
+        api_password:
+            description:
+              - Password or a token of the user to login into OpenNebula RPC server.
+              - If not set, the value of the C(ONE_PASSWORD) environment variable is used.
+            env:
+              - name: ONE_PASSWORD
+            required: False
+            type: string
+        api_authfile:
+            description:
+              - If both I(api_username) or I(api_password) are not set, then it will try
+                authenticate with ONE auth file. Default path is C(~/.one/one_auth).
+              - Set environment variable C(ONE_AUTH) to override this path.
+            env:
+              - name: ONE_AUTH
+            required: False
+            type: string
+        hostname:
+            description: Field to match the hostname. Note C(v4_first_ip) corresponds to the first IPv4 found on VM.
+            type: string
+            default: v4_first_ip
+            choices:
+                - v4_first_ip
+                - v6_first_ip
+                - name
+        filter_by_label:
+            description: Only return servers filtered by this label.
+            type: string
+        group_by_labels:
+            description: Create host groups by vm labels
+            type: bool
+            default: True
+'''
+
+EXAMPLES = r'''
+# inventory_opennebula.yml file in YAML format
+# Example command line: ansible-inventory --list -i inventory_opennebula.yml
+
+# Pass a label filter to the API
+plugin: community.general.opennebula
+api_url: https://opennebula:2633/RPC2
+filter_by_label: Cache
+'''
+
+try:
+    import pyone
+
+    HAS_PYONE = True
+except ImportError:
+    HAS_PYONE = False
+
+from ansible.errors import AnsibleError
+from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
+from ansible.module_utils._text import to_native
+
+from collections import namedtuple
+import os
+
+
+class InventoryModule(BaseInventoryPlugin, Constructable):
+    NAME = 'community.general.opennebula'
+
+    def verify_file(self, path):
+        valid = False
+        if super(InventoryModule, self).verify_file(path):
+            if path.endswith(('opennebula.yaml', 'opennebula.yml')):
+                valid = True
+        return valid
+
+    def _get_connection_info(self):
+        url = self.get_option('api_url')
+        username = self.get_option('api_username')
+        password = self.get_option('api_password')
+        authfile = self.get_option('api_authfile')
+
+        if not username and not password:
+            if authfile is None:
+                authfile = os.path.join(os.environ.get("HOME"), ".one", "one_auth")
+            try:
+                with open(authfile, "r") as fp:
+                    authstring = fp.read().rstrip()
+                username, password = authstring.split(":")
+            except (OSError, IOError):
+                raise AnsibleError("Could not find or read ONE_AUTH file at '{e}'".format(e=authfile))
+            except Exception:
+                raise AnsibleError("Error occurs when reading ONE_AUTH file at '{e}'".format(e=authfile))
+
+        auth_params = namedtuple('auth', ('url', 'username', 'password'))
+
+        return auth_params(url=url, username=username, password=password)
+
+    def _get_vm_ipv4(self, vm):
+        nic = vm.TEMPLATE.get('NIC')
+
+        if isinstance(nic, dict):
+            nic = [nic]
+
+        for net in nic:
+            return net['IP']
+
+        return False
+
+    def _get_vm_ipv6(self, vm):
+        nic = vm.TEMPLATE.get('NIC')
+
+        if isinstance(nic, dict):
+            nic = [nic]
+
+        for net in nic:
+            if net.get('IP6_GLOBAL'):
+                return net['IP6_GLOBAL']
+
+        return False
+
+    def _get_vm_pool(self):
+        auth = self._get_connection_info()
+
+        if not (auth.username and auth.password):
+            raise AnsibleError('API Credentials missing. Check OpenNebula inventory file.')
+        else:
+            one_client = pyone.OneServer(auth.url, session=auth.username + ':' + auth.password)
+
+        # get hosts (VMs)
+        try:
+            vm_pool = one_client.vmpool.infoextended(-2, -1, -1, 3)
+        except Exception as e:
+            raise AnsibleError("Something happened during XML-RPC call: {e}".format(e=to_native(e)))
+
+        return vm_pool
+
+    def _retrieve_servers(self, label_filter=None):
+        vm_pool = self._get_vm_pool()
+
+        result = []
+
+        # iterate over hosts
+        for vm in vm_pool.VM:
+            server = vm.USER_TEMPLATE
+
+            labels = []
+            if vm.USER_TEMPLATE.get('LABELS'):
+                labels = [s for s in vm.USER_TEMPLATE.get('LABELS') if s == ',' or s == '-' or s.isalnum() or s.isspace()]
+                labels = ''.join(labels)
+                labels = labels.replace(' ', '_')
+                labels = labels.replace('-', '_')
+                labels = labels.split(',')
+
+            # filter by label
+            if label_filter is not None:
+                if label_filter not in labels:
+                    continue
+
+            server['name'] = vm.NAME
+            server['LABELS'] = labels
+            server['v4_first_ip'] = self._get_vm_ipv4(vm)
+            server['v6_first_ip'] = self._get_vm_ipv6(vm)
+
+            result.append(server)
+
+        return result
+
+    def _populate(self):
+        hostname_preference = self.get_option('hostname')
+        group_by_labels = self.get_option('group_by_labels')
+
+        # Add a top group 'one'
+        self.inventory.add_group(group='all')
+
+        filter_by_label = self.get_option('filter_by_label')
+        for server in self._retrieve_servers(filter_by_label):
+            # check for labels
+            if group_by_labels and server['LABELS']:
+                for label in server['LABELS']:
+                    self.inventory.add_group(group=label)
+                    self.inventory.add_host(host=server['name'], group=label)
+
+            self.inventory.add_host(host=server['name'], group='all')
+
+            for attribute, value in server.items():
+                self.inventory.set_variable(server['name'], attribute, value)
+
+            if hostname_preference != 'name':
+                self.inventory.set_variable(server['name'], 'ansible_host', server[hostname_preference])
+
+            if server.get('SSH_PORT'):
+                self.inventory.set_variable(server['name'], 'ansible_port', server['SSH_PORT'])
+
+    def parse(self, inventory, loader, path, cache=True):
+        if not HAS_PYONE:
+            raise AnsibleError('OpenNebula Inventory plugin requires pyone to work!')
+
+        super(InventoryModule, self).parse(inventory, loader, path)
+        self._read_config_data(path=path)
+
+        self._populate()

tests/unit/plugins/inventory/test_opennebula.py

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, FELDSAM s.r.o. - FeldHost™ <support@feldhost.cz>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# The API responses used in these tests were recorded from OpenNebula version 5.10.
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+from collections import OrderedDict
+
+import pytest
+
+from ansible.inventory.data import InventoryData
+from ansible_collections.community.general.plugins.inventory.opennebula import InventoryModule
+
+
+@pytest.fixture(scope="module")
+def inventory():
+    r = InventoryModule()
+    r.inventory = InventoryData()
+    return r
+
+
+def test_verify_file(tmp_path, inventory):
+    file = tmp_path / "foobar.opennebula.yml"
+    file.touch()
+    assert inventory.verify_file(str(file)) is True
+
+
+def test_verify_file_bad_config(inventory):
+    assert inventory.verify_file('foobar.opennebula.yml') is False
+
+
+def get_vm_pool():
+    data = type('pyone.bindings.VM_POOLSub', (object,), {'VM': []})()
+
+    vm = type('pyone.bindings.VMType90Sub', (object,), {
+        'DEPLOY_ID': 'one-7157',
+        'ETIME': 0,
+        'GID': 132,
+        'GNAME': 'CSApparelVDC',
+        'HISTORY_RECORDS': {},
+        'ID': 7157,
+        'LAST_POLL': 1632762935,
+        'LCM_STATE': 3,
+        'MONITORING': {},
+        'NAME': 'sam-691-sam',
+        'RESCHED': 0,
+        'SNAPSHOTS': [],
+        'STATE': 3,
+        'STIME': 1632755245,
+        'TEMPLATE': OrderedDict({
+            'NIC': OrderedDict({
+                'AR_ID': '0',
+                'BRIDGE': 'onebr80',
+                'BRIDGE_TYPE': 'linux',
+                'CLUSTER_ID': '0',
+                'IP': '172.22.4.187',
+                'MAC': '02:00:ac:16:04:bb',
+                'MTU': '8192',
+                'NAME': 'NIC0',
+                'NETWORK': 'Private Net CSApparel',
+                'NETWORK_ID': '80',
+                'NETWORK_UNAME': 'CSApparelVDC-admin',
+                'NIC_ID': '0',
+                'PHYDEV': 'team0',
+                'SECURITY_GROUPS': '0',
+                'TARGET': 'one-7157-0',
+                'VLAN_ID': '480',
+                'VN_MAD': '802.1Q'
+            })
+        }),
+        'USER_TEMPLATE': OrderedDict({
+            'HYPERVISOR': 'kvm',
+            'INPUTS_ORDER': '',
+            'LOGO': 'images/logos/centos.png',
+            'MEMORY_UNIT_COST': 'MB',
+            'SCHED_REQUIREMENTS': 'CLUSTER_ID="0"'
+        })
+    })()
+    data.VM.append(vm)
+
+    vm = type('pyone.bindings.VMType90Sub', (object,), {
+        'DEPLOY_ID': 'one-327',
+        'ETIME': 0,
+        'GID': 0,
+        'GNAME': 'oneadmin',
+        'HISTORY_RECORDS': {},
+        'ID': 327,
+        'LAST_POLL': 1632763543,
+        'LCM_STATE': 3,
+        'MONITORING': {},
+        'NAME': 'zabbix-327',
+        'RESCHED': 0,
+        'SNAPSHOTS': [],
+        'STATE': 3,
+        'STIME': 1575410106,
+        'TEMPLATE': OrderedDict({
+            'NIC': [
+                OrderedDict({
+                    'AR_ID': '0',
+                    'BRIDGE': 'onerb.103',
+                    'BRIDGE_TYPE': 'linux',
+                    'IP': '185.165.1.1',
+                    'IP6_GLOBAL': '2000:a001::b9ff:feae:aa0d',
+                    'IP6_LINK': 'fe80::b9ff:feae:aa0d',
+                    'MAC': '02:00:b9:ae:aa:0d',
+                    'NAME': 'NIC0',
+                    'NETWORK': 'Public',
+                    'NETWORK_ID': '7',
+                    'NIC_ID': '0',
+                    'PHYDEV': 'team0',
+                    'SECURITY_GROUPS': '0',
+                    'TARGET': 'one-327-0',
+                    'VLAN_ID': '100',
+                    'VN_MAD': '802.1Q'
+                }),
+                OrderedDict({
+                    'AR_ID': '0',
+                    'BRIDGE': 'br0',
+                    'BRIDGE_TYPE': 'linux',
+                    'CLUSTER_ID': '0',
+                    'IP': '192.168.1.1',
+                    'MAC': '02:00:c0:a8:3b:01',
+                    'NAME': 'NIC1',
+                    'NETWORK': 'Management',
+                    'NETWORK_ID': '11',
+                    'NIC_ID': '1',
+                    'SECURITY_GROUPS': '0',
+                    'TARGET': 'one-327-1',
+                    'VN_MAD': 'bridge'
+                })
+            ]
+        }),
+        'USER_TEMPLATE': OrderedDict({
+            'HYPERVISOR': 'kvm',
+            'INPUTS_ORDER': '',
+            'LABELS': 'Oracle Linux',
+            'LOGO': 'images/logos/centos.png',
+            'MEMORY_UNIT_COST': 'MB',
+            'SAVED_TEMPLATE_ID': '29'
+        })
+    })()
+    data.VM.append(vm)
+
+    vm = type('pyone.bindings.VMType90Sub', (object,), {
+        'DEPLOY_ID': 'one-107',
+        'ETIME': 0,
+        'GID': 0,
+        'GNAME': 'oneadmin',
+        'HISTORY_RECORDS': {},
+        'ID': 107,
+        'LAST_POLL': 1632764186,
+        'LCM_STATE': 3,
+        'MONITORING': {},
+        'NAME': 'gitlab-107',
+        'RESCHED': 0,
+        'SNAPSHOTS': [],
+        'STATE': 3,
+        'STIME': 1572485522,
+        'TEMPLATE': OrderedDict({
+            'NIC': OrderedDict({
+                'AR_ID': '0',
+                'BRIDGE': 'onerb.103',
+                'BRIDGE_TYPE': 'linux',
+                'IP': '185.165.1.3',
+                'IP6_GLOBAL': '2000:a001::b9ff:feae:aa03',
+                'IP6_LINK': 'fe80::b9ff:feae:aa03',
+                'MAC': '02:00:b9:ae:aa:03',
+                'NAME': 'NIC0',
+                'NETWORK': 'Public',
+                'NETWORK_ID': '7',
+                'NIC_ID': '0',
+                'PHYDEV': 'team0',
+                'SECURITY_GROUPS': '0',
+                'TARGET': 'one-107-0',
+                'VLAN_ID': '100',
+                'VN_MAD': '802.1Q'
+            })
+        }),
+        'USER_TEMPLATE': OrderedDict({
+            'HYPERVISOR': 'kvm',
+            'INPUTS_ORDER': '',
+            'LABELS': 'Gitlab,Centos',
+            'LOGO': 'images/logos/centos.png',
+            'MEMORY_UNIT_COST': 'MB',
+            'SCHED_REQUIREMENTS': 'ID="0" | ID="1" | ID="2"',
+            'SSH_PORT': '8822'
+        })
+    })()
+    data.VM.append(vm)
+
+    return data
+
+
+def get_option(option):
+    if option == 'api_url':
+        return 'https://opennebula:2633/RPC2'
+    if option == 'api_username':
+        return 'username'
+    elif option == 'api_password':
+        return 'password'
+    elif option == 'api_authfile':
+        return '~/.one/one_auth'
+    elif option == 'hostname':
+        return 'v4_first_ip'
+    elif option == 'group_by_labels':
+        return True
+    elif option == 'filter_by_label':
+        return None
+    else:
+        return False
+
+
+def test_get_connection_info(inventory, mocker):
+    inventory.get_option = mocker.MagicMock(side_effect=get_option)
+
+    auth = inventory._get_connection_info()
+    assert (auth.username and auth.password)
+
+
+def test_populate(inventory, mocker):
+    # bypass API fetch call
+    inventory._get_vm_pool = mocker.MagicMock(side_effect=get_vm_pool)
+    inventory.get_option = mocker.MagicMock(side_effect=get_option)
+    inventory._populate()
+
+    # get different hosts
+    host_sam = inventory.inventory.get_host('sam-691-sam')
+    host_zabbix = inventory.inventory.get_host('zabbix-327')
+    host_gitlab = inventory.inventory.get_host('gitlab-107')
+
+    # test if groups exists
+    assert 'Gitlab' in inventory.inventory.groups
+    assert 'Centos' in inventory.inventory.groups
+    assert 'Oracle_Linux' in inventory.inventory.groups
+
+    # check if host_zabbix is in Oracle_Linux group
+    group_oracle_linux = inventory.inventory.groups['Oracle_Linux']
+    assert group_oracle_linux.hosts == [host_zabbix]
+
+    # check if host_gitlab is in Gitlab and Centos group
+    group_gitlab = inventory.inventory.groups['Gitlab']
+    group_centos = inventory.inventory.groups['Centos']
+    assert group_gitlab.hosts == [host_gitlab]
+    assert group_centos.hosts == [host_gitlab]
+
+    # check IPv4 address
+    assert '172.22.4.187' == host_sam.get_vars()['v4_first_ip']
+
+    # check IPv6 address
+    assert '2000:a001::b9ff:feae:aa0d' == host_zabbix.get_vars()['v6_first_ip']
+
+    # check ansible_hosts
+    assert '172.22.4.187' == host_sam.get_vars()['ansible_host']
+    assert '185.165.1.1' == host_zabbix.get_vars()['ansible_host']
+    assert '185.165.1.3' == host_gitlab.get_vars()['ansible_host']
+
+    # check for custom ssh port
+    assert '8822' == host_gitlab.get_vars()['ansible_port']