From 73be912bf7dce4dbc46ca2eda40971a019102de1 Mon Sep 17 00:00:00 2001 From: Jeffrey van Pelt Date: Fri, 21 Aug 2020 13:16:59 +0200 Subject: [PATCH] New inventory module: Proxmox (#545) * This commit adds proxmox inventory module and proxmox_snap for snapshot management * Fixed pylint errors * Missed this one.. * This should fix the doc errors * Remove proxmox_snap to allow for single module per PR * Changes as suggested by felixfontein in #535 * Reverted back to AnsibleError as module.fail_json broke it. Need to investigate further * Made importerror behave similar to docker_swarm and gitlab_runner * FALSE != False * Added myself as author * Added a requested feature from a colleague to also sort VMs based on their running state * Prevent VM templates from being added to the inventory * Processed feedback * Updated my email and included version * Processed doc feedback * More feedback processed * Shortened this line of documentation, it is a duplicate and it was causing a sanity error (> 160 characters) * Added test from PR #736 to check what needs to be changed to make it work * Changed some tests around * Remove some tests, first get these working * Disabled all tests, except the one I am hacking together now * Added mocker, still trying to figure this out * Am I looking in the right direction? * Processed docs feedback * Fixed bot feedback * Removed all other tests, started with basic ones (borrowed from cobbler) * Removed all other tests, started with basic ones (borrowed from cobbler) * Removed all other tests, started with basic ones (borrowed from cobbler) * Removed init_cache test as it is implemented on a different way in the original foreman/satellite inventory (and thus also this one) * This actually passes! Need to check if I need to add asserts as well * Made bot happy again? * Added some assertions * Added note about PVE API version * Mocked only get_json, the rest functions as-is * Fixed sanity errors * Fixed version bump (again...) ;-) * Processed feedback --- plugins/inventory/proxmox.py | 348 +++++++++++++++++++ tests/unit/plugins/inventory/test_proxmox.py | 187 ++++++++++ 2 files changed, 535 insertions(+) create mode 100644 plugins/inventory/proxmox.py create mode 100644 tests/unit/plugins/inventory/test_proxmox.py diff --git a/plugins/inventory/proxmox.py b/plugins/inventory/proxmox.py new file mode 100644 index 0000000000..09e00d34cd --- /dev/null +++ b/plugins/inventory/proxmox.py @@ -0,0 +1,348 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016 Guido Günther , Daniel Lobato Garcia +# Copyright (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: proxmox + plugin_type: inventory + short_description: Proxmox inventory source + version_added: "1.2.0" + author: + - Jeffrey van Pelt (@Thulium-Drake) + requirements: + - requests >= 1.1 + description: + - Get inventory hosts from a Proxmox PVE cluster. + - "Uses a configuration file as an inventory source, it must end in C(.proxmox.yml) or C(.proxmox.yaml)" + - Will retrieve the first network interface with an IP for Proxmox nodes. + - Can retrieve LXC/QEMU configuration as facts. + extends_documentation_fragment: + - inventory_cache + options: + plugin: + description: The name of this plugin, it should always be set to C(community.general.proxmox) for this plugin to recognize it as it's own. + required: yes + choices: ['community.general.proxmox'] + type: str + url: + description: URL to Proxmox cluster. + default: 'http://localhost:8006' + type: str + user: + description: Proxmox authentication user. + required: yes + type: str + password: + description: Proxmox authentication password. + required: yes + type: str + validate_certs: + description: Verify SSL certificate if using HTTPS. + type: boolean + default: yes + group_prefix: + description: Prefix to apply to Proxmox groups. + default: proxmox_ + type: str + facts_prefix: + description: Prefix to apply to LXC/QEMU config facts. + default: proxmox_ + type: str + want_facts: + description: Gather LXC/QEMU configuration facts. + default: no + type: bool +''' + +EXAMPLES = ''' +# my.proxmox.yml +plugin: community.general.proxmox +url: http://localhost:8006 +user: ansible@pve +password: secure +validate_certs: no +''' + +import re + +from ansible.module_utils.common._collections_compat import MutableMapping +from distutils.version import LooseVersion + +from ansible.errors import AnsibleError +from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable +from ansible.module_utils.six.moves.urllib.parse import urlencode + +# 3rd party imports +try: + import requests + if LooseVersion(requests.__version__) < LooseVersion('1.1.0'): + raise ImportError + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + + +class InventoryModule(BaseInventoryPlugin, Cacheable): + ''' Host inventory parser for ansible using Proxmox as source. ''' + + NAME = 'community.general.proxmox' + + def __init__(self): + + super(InventoryModule, self).__init__() + + # from config + self.proxmox_url = None + + self.session = None + self.cache_key = None + self.use_cache = None + + def verify_file(self, path): + + valid = False + if super(InventoryModule, self).verify_file(path): + if path.endswith(('proxmox.yaml', 'proxmox.yml')): + valid = True + else: + self.display.vvv('Skipping due to inventory source not ending in "proxmox.yaml" nor "proxmox.yml"') + return valid + + def _get_session(self): + if not self.session: + self.session = requests.session() + self.session.verify = self.get_option('validate_certs') + return self.session + + def _get_auth(self): + credentials = urlencode({'username': self.proxmox_user, 'password': self.proxmox_password, }) + + a = self._get_session() + ret = a.post('%s/api2/json/access/ticket' % self.proxmox_url, data=credentials) + + json = ret.json() + + self.credentials = { + 'ticket': json['data']['ticket'], + 'CSRFPreventionToken': json['data']['CSRFPreventionToken'], + } + + def _get_json(self, url, ignore_errors=None): + + if not self.use_cache or url not in self._cache.get(self.cache_key, {}): + + if self.cache_key not in self._cache: + self._cache[self.cache_key] = {'url': ''} + + data = [] + s = self._get_session() + while True: + headers = {'Cookie': 'PVEAuthCookie={0}'.format(self.credentials['ticket'])} + ret = s.get(url, headers=headers) + if ignore_errors and ret.status_code in ignore_errors: + break + ret.raise_for_status() + json = ret.json() + + # process results + # FIXME: This assumes 'return type' matches a specific query, + # it will break if we expand the queries and they dont have different types + if 'data' not in json: + # /hosts/:id does not have a 'data' key + data = json + break + elif isinstance(json['data'], MutableMapping): + # /facts are returned as dict in 'data' + data = json['data'] + break + else: + # /hosts 's 'results' is a list of all hosts, returned is paginated + data = data + json['data'] + break + + self._cache[self.cache_key][url] = data + + return self._cache[self.cache_key][url] + + def _get_nodes(self): + return self._get_json("%s/api2/json/nodes" % self.proxmox_url) + + def _get_pools(self): + return self._get_json("%s/api2/json/pools" % self.proxmox_url) + + def _get_lxc_per_node(self, node): + return self._get_json("%s/api2/json/nodes/%s/lxc" % (self.proxmox_url, node)) + + def _get_qemu_per_node(self, node): + return self._get_json("%s/api2/json/nodes/%s/qemu" % (self.proxmox_url, node)) + + def _get_members_per_pool(self, pool): + ret = self._get_json("%s/api2/json/pools/%s" % (self.proxmox_url, pool)) + return ret['members'] + + def _get_node_ip(self, node): + ret = self._get_json("%s/api2/json/nodes/%s/network" % (self.proxmox_url, node)) + + for iface in ret: + try: + return iface['address'] + except Exception: + return None + + def _get_vm_config(self, node, vmid, vmtype, name): + ret = self._get_json("%s/api2/json/nodes/%s/%s/%s/config" % (self.proxmox_url, node, vmtype, vmid)) + + vmid_key = 'vmid' + vmid_key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), vmid_key.lower())) + self.inventory.set_variable(name, vmid_key, vmid) + + vmtype_key = 'vmtype' + vmtype_key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), vmtype_key.lower())) + self.inventory.set_variable(name, vmtype_key, vmtype) + + for config in ret: + key = config + key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), key.lower())) + value = ret[config] + try: + # fixup disk images as they have no key + if config == 'rootfs' or config.startswith(('virtio', 'sata', 'ide', 'scsi')): + value = ('disk_image=' + value) + + if isinstance(value, int) or ',' not in value: + value = value + # split off strings with commas to a dict + else: + # skip over any keys that cannot be processed + try: + value = dict(key.split("=") for key in value.split(",")) + except Exception: + continue + + self.inventory.set_variable(name, key, value) + except NameError: + return None + + def _get_vm_status(self, node, vmid, vmtype, name): + ret = self._get_json("%s/api2/json/nodes/%s/%s/%s/status/current" % (self.proxmox_url, node, vmtype, vmid)) + + status = ret['status'] + status_key = 'status' + status_key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), status_key.lower())) + self.inventory.set_variable(name, status_key, status) + + def to_safe(self, word): + '''Converts 'bad' characters in a string to underscores so they can be used as Ansible groups + #> ProxmoxInventory.to_safe("foo-bar baz") + 'foo_barbaz' + ''' + regex = r"[^A-Za-z0-9\_]" + return re.sub(regex, "_", word.replace(" ", "")) + + def _populate(self): + + self._get_auth() + + # gather vm's on nodes + for node in self._get_nodes(): + # FIXME: this can probably be cleaner + # create groups + lxc_group = 'all_lxc' + lxc_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), lxc_group.lower())) + self.inventory.add_group(lxc_group) + qemu_group = 'all_qemu' + qemu_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), qemu_group.lower())) + self.inventory.add_group(qemu_group) + nodes_group = 'nodes' + nodes_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), nodes_group.lower())) + self.inventory.add_group(nodes_group) + running_group = 'all_running' + running_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), running_group.lower())) + self.inventory.add_group(running_group) + stopped_group = 'all_stopped' + stopped_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), stopped_group.lower())) + self.inventory.add_group(stopped_group) + + if node.get('node'): + self.inventory.add_host(node['node']) + + if node['type'] == 'node': + self.inventory.add_child(nodes_group, node['node']) + + # get node IP address + ip = self._get_node_ip(node['node']) + self.inventory.set_variable(node['node'], 'ansible_host', ip) + + # get LXC containers for this node + node_lxc_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), ('%s_lxc' % node['node']).lower())) + self.inventory.add_group(node_lxc_group) + for lxc in self._get_lxc_per_node(node['node']): + self.inventory.add_host(lxc['name']) + self.inventory.add_child(lxc_group, lxc['name']) + self.inventory.add_child(node_lxc_group, lxc['name']) + + # get LXC status when want_facts == True + if self.get_option('want_facts'): + self._get_vm_status(node['node'], lxc['vmid'], 'lxc', lxc['name']) + if lxc['status'] == 'stopped': + self.inventory.add_child(stopped_group, lxc['name']) + elif lxc['status'] == 'running': + self.inventory.add_child(running_group, lxc['name']) + + # get LXC config for facts + if self.get_option('want_facts'): + self._get_vm_config(node['node'], lxc['vmid'], 'lxc', lxc['name']) + + # get QEMU vm's for this node + node_qemu_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), ('%s_qemu' % node['node']).lower())) + self.inventory.add_group(node_qemu_group) + for qemu in self._get_qemu_per_node(node['node']): + if not qemu['template']: + self.inventory.add_host(qemu['name']) + self.inventory.add_child(qemu_group, qemu['name']) + self.inventory.add_child(node_qemu_group, qemu['name']) + + # get QEMU status + self._get_vm_status(node['node'], qemu['vmid'], 'qemu', qemu['name']) + if qemu['status'] == 'stopped': + self.inventory.add_child(stopped_group, qemu['name']) + elif qemu['status'] == 'running': + self.inventory.add_child(running_group, qemu['name']) + + # get QEMU config for facts + if self.get_option('want_facts'): + self._get_vm_config(node['node'], qemu['vmid'], 'qemu', qemu['name']) + + # gather vm's in pools + for pool in self._get_pools(): + if pool.get('poolid'): + pool_group = 'pool_' + pool['poolid'] + pool_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), pool_group.lower())) + self.inventory.add_group(pool_group) + + for member in self._get_members_per_pool(pool['poolid']): + if member.get('name'): + self.inventory.add_child(pool_group, member['name']) + + def parse(self, inventory, loader, path, cache=True): + if not HAS_REQUESTS: + raise AnsibleError('This module requires Python Requests 1.1.0 or higher: ' + 'https://github.com/psf/requests.') + + super(InventoryModule, self).parse(inventory, loader, path) + + # read config from file, this sets 'options' + self._read_config_data(path) + + # get connection host + self.proxmox_url = self.get_option('url') + self.proxmox_user = self.get_option('user') + self.proxmox_password = self.get_option('password') + self.cache_key = self.get_cache_key(path) + self.use_cache = cache and self.get_option('cache') + + # actually populate inventory + self._populate() diff --git a/tests/unit/plugins/inventory/test_proxmox.py b/tests/unit/plugins/inventory/test_proxmox.py new file mode 100644 index 0000000000..e03d1b8013 --- /dev/null +++ b/tests/unit/plugins/inventory/test_proxmox.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Jeffrey van Pelt +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# +# The API responses used in these tests were recorded from PVE version 6.2. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import pytest + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.inventory.data import InventoryData +from ansible_collections.community.general.plugins.inventory.proxmox import InventoryModule + + +@pytest.fixture(scope="module") +def inventory(): + r = InventoryModule() + r.inventory = InventoryData() + return r + + +def test_verify_file_bad_config(inventory): + assert inventory.verify_file('foobar.proxmox.yml') is False + + +def get_auth(): + return True + + +# NOTE: when updating/adding replies to this function, +# be sure to only add only the _contents_ of the 'data' dict in the API reply +def get_json(url): + if url == "https://localhost:8006/api2/json/nodes": + # _get_nodes + return [{"type": "node", + "cpu": 0.01, + "maxdisk": 500, + "mem": 500, + "node": "testnode", + "id": "node/testnode", + "maxcpu": 1, + "status": "online", + "ssl_fingerprint": "xx", + "disk": 1000, + "maxmem": 1000, + "uptime": 10000, + "level": ""}] + elif url == "https://localhost:8006/api2/json/pools": + # _get_pools + return [{"poolid": "test"}] + elif url == "https://localhost:8006/api2/json/nodes/testnode/lxc": + # _get_lxc_per_node + return [{"cpus": 1, + "name": "test-lxc", + "cpu": 0.01, + "diskwrite": 0, + "lock": "", + "maxmem": 1000, + "template": "", + "diskread": 0, + "mem": 1000, + "swap": 0, + "type": "lxc", + "maxswap": 0, + "maxdisk": "1000", + "netout": 1000, + "pid": "1000", + "netin": 1000, + "status": "running", + "vmid": "100", + "disk": "1000", + "uptime": 1000}] + elif url == "https://localhost:8006/api2/json/nodes/testnode/qemu": + # _get_qemu_per_node + return [{"name": "test-qemu", + "cpus": 1, + "mem": 1000, + "template": "", + "diskread": 0, + "cpu": 0.01, + "maxmem": 1000, + "diskwrite": 0, + "netout": 1000, + "pid": "1001", + "netin": 1000, + "maxdisk": 1000, + "vmid": "101", + "uptime": 1000, + "disk": 0, + "status": "running"}] + elif url == "https://localhost:8006/api2/json/pools/test": + # _get_members_per_pool + return {"members": [{"uptime": 1000, + "template": 0, + "id": "qemu/101", + "mem": 1000, + "status": "running", + "cpu": 0.01, + "maxmem": 1000, + "diskwrite": 1000, + "name": "test-qemu", + "netout": 1000, + "netin": 1000, + "vmid": 101, + "node": "testnode", + "maxcpu": 1, + "type": "qemu", + "maxdisk": 1000, + "disk": 0, + "diskread": 1000}]} + elif url == "https://localhost:8006/api2/json/nodes/testnode/network": + # _get_node_ip + return [{"families": ["inet"], + "priority": 3, + "active": 1, + "cidr": "10.1.1.2/24", + "iface": "eth0", + "method": "static", + "exists": 1, + "type": "eth", + "netmask": "24", + "gateway": "10.1.1.1", + "address": "10.1.1.2", + "method6": "manual", + "autostart": 1}, + {"method6": "manual", + "autostart": 1, + "type": "OVSPort", + "exists": 1, + "method": "manual", + "iface": "eth1", + "ovs_bridge": "vmbr0", + "active": 1, + "families": ["inet"], + "priority": 5, + "ovs_type": "OVSPort"}, + {"type": "OVSBridge", + "method": "manual", + "iface": "vmbr0", + "families": ["inet"], + "priority": 4, + "ovs_ports": "eth1", + "ovs_type": "OVSBridge", + "method6": "manual", + "autostart": 1, + "active": 1}] + + +def get_vm_status(node, vmtype, vmid, name): + return True + + +def get_option(option): + if option == 'group_prefix': + return 'proxmox_' + else: + return False + + +def test_populate(inventory, mocker): + # module settings + inventory.proxmox_user = 'root@pam' + inventory.proxmox_password = 'password' + inventory.proxmox_url = 'https://localhost:8006' + + # bypass authentication and API fetch calls + inventory._get_auth = mocker.MagicMock(side_effect=get_auth) + inventory._get_json = mocker.MagicMock(side_effect=get_json) + inventory._get_vm_status = mocker.MagicMock(side_effect=get_vm_status) + inventory.get_option = mocker.MagicMock(side_effect=get_option) + inventory._populate() + + # get different hosts + host_qemu = inventory.inventory.get_host('test-qemu') + host_lxc = inventory.inventory.get_host('test-lxc') + host_node = inventory.inventory.get_host('testnode') + + # check if qemu-test is in the proxmox_pool_test group + assert 'proxmox_pool_test' in inventory.inventory.groups + group_qemu = inventory.inventory.groups['proxmox_pool_test'] + assert group_qemu.hosts == [host_qemu] + + # check if lxc-test has been discovered correctly + group_lxc = inventory.inventory.groups['proxmox_all_lxc'] + assert group_lxc.hosts == [host_lxc]