New Proxmox VE modules to handle pools and their membership (#6604)

* New Proxmox VE modules to handle pools and their membership

* Fix pep8 linting errors

* Fix pep8 and compatibility errors

* Add required fields in the documentation

* Typo fix

* Fix pylint errors

* Fix the last one error

* Address review comments

* Fix linting error

* Add integration tests playbook

* Add assert for the diff mode

* Address review comments

* Fix typo in the word

* Fail for non-empty pool even in check_mode

.github/BOTMETA.yml

@@ -981,7 +981,7 @@ files:
   $modules/proxmox:
     keywords: kvm libvirt proxmox qemu
     labels: proxmox virt
-    maintainers: $team_virt
+    maintainers: $team_virt UnderGreen
   $modules/proxmox.py:
     ignore: skvidal
     maintainers: UnderGreen

plugins/module_utils/proxmox.py

@@ -145,3 +145,25 @@ class ProxmoxAnsible(object):
     def api_task_ok(self, node, taskid):
         status = self.proxmox_api.nodes(node).tasks(taskid).status.get()
         return status['status'] == 'stopped' and status['exitstatus'] == 'OK'
+
+    def get_pool(self, poolid):
+        """Retrieve pool information
+
+        :param poolid: str - name of the pool
+        :return: dict - pool information
+        """
+        try:
+            return self.proxmox_api.pools(poolid).get()
+        except Exception as e:
+            self.module.fail_json(msg="Unable to retrieve pool %s information: %s" % (poolid, e))
+
+    def get_storages(self, type):
+        """Retrieve storages information
+
+        :param type: str, optional - type of storages
+        :return: list of dicts - array of storages
+        """
+        try:
+            return self.proxmox_api.storage.get(type=type)
+        except Exception as e:
+            self.module.fail_json(msg="Unable to retrieve storages information with type %s: %s" % (type, e))

plugins/modules/proxmox_pool.py

@@ -0,0 +1,180 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023, Sergei Antipov (UnderGreen) <greendayonfire@gmail.com>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = r"""
+---
+module: proxmox_pool
+short_description: Pool management for Proxmox VE cluster
+description:
+  - Create or delete a pool for Proxmox VE clusters.
+  - For pool members management please consult M(community.general.proxmox_pool_member) module.
+version_added: 7.1.0
+author: "Sergei Antipov (@UnderGreen) <greendayonfire@gmail.com>"
+attributes:
+  check_mode:
+    support: full
+  diff_mode:
+    support: none
+options:
+  poolid:
+    description:
+      - The pool ID.
+    type: str
+    aliases: [ "name" ]
+    required: true
+  state:
+    description:
+     - Indicate desired state of the pool.
+     - The pool must be empty prior deleting it with O(state=absent).
+    choices: ['present', 'absent']
+    default: present
+    type: str
+  comment:
+    description:
+      - Specify the description for the pool.
+      - Parameter is ignored when pool already exists or O(state=absent).
+    type: str
+
+extends_documentation_fragment:
+    - community.general.proxmox.documentation
+    - community.general.attributes
+"""
+
+EXAMPLES = """
+- name: Create new Proxmox VE pool
+  community.general.proxmox_pool:
+    api_host: node1
+    api_user: root@pam
+    api_password: password
+    poolid: test
+    comment: 'New pool'
+
+- name: Delete the Proxmox VE pool
+  community.general.proxmox_pool:
+    api_host: node1
+    api_user: root@pam
+    api_password: password
+    poolid: test
+    state: absent
+"""
+
+RETURN = """
+poolid:
+  description: The pool ID.
+  returned: success
+  type: str
+  sample: test
+msg:
+  description: A short message on what the module did.
+  returned: always
+  type: str
+  sample: "Pool test successfully created"
+"""
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.community.general.plugins.module_utils.proxmox import (proxmox_auth_argument_spec, ProxmoxAnsible)
+
+
+class ProxmoxPoolAnsible(ProxmoxAnsible):
+
+    def is_pool_existing(self, poolid):
+        """Check whether pool already exist
+
+        :param poolid: str - name of the pool
+        :return: bool - is pool exists?
+        """
+        try:
+            pools = self.proxmox_api.pools.get()
+            for pool in pools:
+                if pool['poolid'] == poolid:
+                    return True
+            return False
+        except Exception as e:
+            self.module.fail_json(msg="Unable to retrieve pools: {0}".format(e))
+
+    def is_pool_empty(self, poolid):
+        """Check whether pool has members
+
+        :param poolid: str - name of the pool
+        :return: bool - is pool empty?
+        """
+        return True if not self.get_pool(poolid)['members'] else False
+
+    def create_pool(self, poolid, comment=None):
+        """Create Proxmox VE pool
+
+        :param poolid: str - name of the pool
+        :param comment: str, optional - Description of a pool
+        :return: None
+        """
+        if self.is_pool_existing(poolid):
+            self.module.exit_json(changed=False, poolid=poolid, msg="Pool {0} already exists".format(poolid))
+
+        if self.module.check_mode:
+            return
+
+        try:
+            self.proxmox_api.pools.post(poolid=poolid, comment=comment)
+        except Exception as e:
+            self.module.fail_json(msg="Failed to create pool with ID {0}: {1}".format(poolid, e))
+
+    def delete_pool(self, poolid):
+        """Delete Proxmox VE pool
+
+        :param poolid: str - name of the pool
+        :return: None
+        """
+        if not self.is_pool_existing(poolid):
+            self.module.exit_json(changed=False, poolid=poolid, msg="Pool {0} doesn't exist".format(poolid))
+
+        if self.is_pool_empty(poolid):
+            if self.module.check_mode:
+                return
+
+            try:
+                self.proxmox_api.pools(poolid).delete()
+            except Exception as e:
+                self.module.fail_json(msg="Failed to delete pool with ID {0}: {1}".format(poolid, e))
+        else:
+            self.module.fail_json(msg="Can't delete pool {0} with members. Please remove members from pool first.".format(poolid))
+
+
+def main():
+    module_args = proxmox_auth_argument_spec()
+    pools_args = dict(
+        poolid=dict(type="str", aliases=["name"], required=True),
+        comment=dict(type="str"),
+        state=dict(default="present", choices=["present", "absent"]),
+    )
+
+    module_args.update(pools_args)
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        required_together=[("api_token_id", "api_token_secret")],
+        required_one_of=[("api_password", "api_token_id")],
+        supports_check_mode=True
+    )
+
+    poolid = module.params["poolid"]
+    comment = module.params["comment"]
+    state = module.params["state"]
+
+    proxmox = ProxmoxPoolAnsible(module)
+
+    if state == "present":
+        proxmox.create_pool(poolid, comment)
+        module.exit_json(changed=True, poolid=poolid, msg="Pool {0} successfully created".format(poolid))
+    else:
+        proxmox.delete_pool(poolid)
+        module.exit_json(changed=True, poolid=poolid, msg="Pool {0} successfully deleted".format(poolid))
+
+
+if __name__ == "__main__":
+    main()

plugins/modules/proxmox_pool_member.py

@@ -0,0 +1,238 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023, Sergei Antipov (UnderGreen) <greendayonfire@gmail.com>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = r"""
+---
+module: proxmox_pool_member
+short_description: Add or delete members from Proxmox VE cluster pools
+description:
+  - Create or delete a pool member in Proxmox VE clusters.
+version_added: 7.1.0
+author: "Sergei Antipov (@UnderGreen) <greendayonfire@gmail.com>"
+attributes:
+  check_mode:
+    support: full
+  diff_mode:
+    support: full
+options:
+  poolid:
+    description:
+      - The pool ID.
+    type: str
+    aliases: [ "name" ]
+    required: true
+  member:
+    description:
+      - Specify the member name.
+      - For O(type=storage) it is a storage name.
+      - For O(type=vm) either vmid or vm name could be used.
+    type: str
+    required: true
+  type:
+    description:
+      - Member type to add/remove from the pool.
+    choices: ["vm", "storage"]
+    default: vm
+    type: str
+  state:
+    description:
+     - Indicate desired state of the pool member.
+    choices: ['present', 'absent']
+    default: present
+    type: str
+
+extends_documentation_fragment:
+    - community.general.proxmox.documentation
+    - community.general.attributes
+"""
+
+EXAMPLES = """
+- name: Add new VM to Proxmox VE pool
+  community.general.proxmox_pool_member:
+    api_host: node1
+    api_user: root@pam
+    api_password: password
+    poolid: test
+    member: 101
+
+- name: Add new storage to Proxmox VE pool
+  community.general.proxmox_pool_member:
+    api_host: node1
+    api_user: root@pam
+    api_password: password
+    poolid: test
+    member: zfs-data
+    type: storage
+
+- name: Remove VM from the Proxmox VE pool using VM name
+  community.general.proxmox_pool_member:
+    api_host: node1
+    api_user: root@pam
+    api_password: password
+    poolid: test
+    member: pxe.home.arpa
+    state: absent
+
+- name: Remove storage from the Proxmox VE pool
+  community.general.proxmox_pool_member:
+    api_host: node1
+    api_user: root@pam
+    api_password: password
+    poolid: test
+    member: zfs-storage
+    type: storage
+    state: absent
+"""
+
+RETURN = """
+poolid:
+  description: The pool ID.
+  returned: success
+  type: str
+  sample: test
+member:
+  description: Member name.
+  returned: success
+  type: str
+  sample: 101
+msg:
+  description: A short message on what the module did.
+  returned: always
+  type: str
+  sample: "Member 101 deleted from the pool test"
+"""
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.community.general.plugins.module_utils.proxmox import (proxmox_auth_argument_spec, ProxmoxAnsible)
+
+
+class ProxmoxPoolMemberAnsible(ProxmoxAnsible):
+
+    def pool_members(self, poolid):
+        vms = []
+        storage = []
+        for member in self.get_pool(poolid)["members"]:
+            if member["type"] == "storage":
+                storage.append(member["storage"])
+            else:
+                vms.append(member["vmid"])
+
+        return (vms, storage)
+
+    def add_pool_member(self, poolid, member, member_type):
+        current_vms_members, current_storage_members = self.pool_members(poolid)
+        all_members_before = current_storage_members + current_vms_members
+        all_members_after = all_members_before.copy()
+        diff = {"before": {"members": all_members_before}, "after": {"members": all_members_after}}
+
+        try:
+            if member_type == "storage":
+                storages = self.get_storages(type=None)
+                if member not in [storage["storage"] for storage in storages]:
+                    self.module.fail_json(msg="Storage {0} doesn't exist in the cluster".format(member))
+                if member in current_storage_members:
+                    self.module.exit_json(changed=False, poolid=poolid, member=member,
+                                          diff=diff, msg="Member {0} is already part of the pool {1}".format(member, poolid))
+
+                all_members_after.append(member)
+                if self.module.check_mode:
+                    return diff
+
+                self.proxmox_api.pools(poolid).put(storage=[member])
+                return diff
+            else:
+                try:
+                    vmid = int(member)
+                except ValueError:
+                    vmid = self.get_vmid(member)
+
+                if vmid in current_vms_members:
+                    self.module.exit_json(changed=False, poolid=poolid, member=member,
+                                          diff=diff, msg="VM {0} is already part of the pool {1}".format(member, poolid))
+
+                all_members_after.append(member)
+
+                if not self.module.check_mode:
+                    self.proxmox_api.pools(poolid).put(vms=[vmid])
+                return diff
+        except Exception as e:
+            self.module.fail_json(msg="Failed to add a new member ({0}) to the pool {1}: {2}".format(member, poolid, e))
+
+    def delete_pool_member(self, poolid, member, member_type):
+        current_vms_members, current_storage_members = self.pool_members(poolid)
+        all_members_before = current_storage_members + current_vms_members
+        all_members_after = all_members_before.copy()
+        diff = {"before": {"members": all_members_before}, "after": {"members": all_members_after}}
+
+        try:
+            if member_type == "storage":
+                if member not in current_storage_members:
+                    self.module.exit_json(changed=False, poolid=poolid, member=member,
+                                          diff=diff, msg="Member {0} is not part of the pool {1}".format(member, poolid))
+
+                all_members_after.remove(member)
+                if self.module.check_mode:
+                    return diff
+
+                self.proxmox_api.pools(poolid).put(storage=[member], delete=1)
+                return diff
+            else:
+                try:
+                    vmid = int(member)
+                except ValueError:
+                    vmid = self.get_vmid(member)
+
+                if vmid not in current_vms_members:
+                    self.module.exit_json(changed=False, poolid=poolid, member=member,
+                                          diff=diff, msg="VM {0} is not part of the pool {1}".format(member, poolid))
+
+                all_members_after.remove(member)
+
+                if not self.module.check_mode:
+                    self.proxmox_api.pools(poolid).put(vms=[vmid], delete=1)
+                return diff
+        except Exception as e:
+            self.module.fail_json(msg="Failed to delete a member ({0}) from the pool {1}: {2}".format(member, poolid, e))
+
+
+def main():
+    module_args = proxmox_auth_argument_spec()
+    pool_members_args = dict(
+        poolid=dict(type="str", aliases=["name"], required=True),
+        member=dict(type="str", required=True),
+        type=dict(default="vm", choices=["vm", "storage"]),
+        state=dict(default="present", choices=["present", "absent"]),
+    )
+
+    module_args.update(pool_members_args)
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        required_together=[("api_token_id", "api_token_secret")],
+        required_one_of=[("api_password", "api_token_id")],
+        supports_check_mode=True
+    )
+
+    poolid = module.params["poolid"]
+    member = module.params["member"]
+    member_type = module.params["type"]
+    state = module.params["state"]
+
+    proxmox = ProxmoxPoolMemberAnsible(module)
+
+    if state == "present":
+        diff = proxmox.add_pool_member(poolid, member, member_type)
+        module.exit_json(changed=True, poolid=poolid, member=member, diff=diff, msg="New member {0} added to the pool {1}".format(member, poolid))
+    else:
+        diff = proxmox.delete_pool_member(poolid, member, member_type)
+        module.exit_json(changed=True, poolid=poolid, member=member, diff=diff, msg="Member {0} deleted from the pool {1}".format(member, poolid))
+
+
+if __name__ == "__main__":
+    main()

tests/integration/targets/proxmox_pool/aliases

@@ -0,0 +1,7 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+unsupported
+proxmox_pool
+proxmox_pool_member

tests/integration/targets/proxmox_pool/defaults/main.yml

@@ -0,0 +1,7 @@
+# Copyright (c) 2023, Sergei Antipov <greendayonfire at gmail.com>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+poolid: test
+member: local
+member_type: storage

tests/integration/targets/proxmox_pool/tasks/main.yml

@@ -0,0 +1,220 @@
+####################################################################
+# WARNING: These are designed specifically for Ansible tests       #
+# and should not be used as examples of how to write Ansible roles #
+####################################################################
+
+# Copyright (c) 2023, Sergei Antipov <greendayonfire at gmail.com>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Proxmox VE pool and pool membership management
+  tags: ["pool"]
+  block:
+  - name: Make sure poolid parameter is not missing
+    proxmox_pool:
+      api_host: "{{ api_host }}"
+      api_user: "{{ user }}@{{ domain }}"
+      api_password: "{{ api_password | default(omit) }}"
+      api_token_id: "{{ api_token_id | default(omit) }}"
+      api_token_secret: "{{ api_token_secret | default(omit) }}"
+      validate_certs: "{{ validate_certs }}"
+    ignore_errors: true
+    register: result
+
+  - assert:
+      that:
+        - result is failed
+        - "'missing required arguments: poolid' in result.msg"
+
+  - name: Create pool (Check)
+    proxmox_pool:
+      api_host: "{{ api_host }}"
+      api_user: "{{ user }}@{{ domain }}"
+      api_password: "{{ api_password | default(omit) }}"
+      api_token_id: "{{ api_token_id | default(omit) }}"
+      api_token_secret: "{{ api_token_secret | default(omit) }}"
+      validate_certs: "{{ validate_certs }}"
+      poolid: "{{ poolid }}"
+    check_mode: true
+    register: result
+
+  - assert:
+      that:
+        - result is changed
+        - result is success
+
+  - name: Create pool
+    proxmox_pool:
+      api_host: "{{ api_host }}"
+      api_user: "{{ user }}@{{ domain }}"
+      api_password: "{{ api_password | default(omit) }}"
+      api_token_id: "{{ api_token_id | default(omit) }}"
+      api_token_secret: "{{ api_token_secret | default(omit) }}"
+      validate_certs: "{{ validate_certs }}"
+      poolid: "{{ poolid }}"
+    register: result
+
+  - assert:
+      that:
+        - result is changed
+        - result is success
+        - result.poolid == "{{ poolid }}"
+
+  - name: Delete pool (Check)
+    proxmox_pool:
+      api_host: "{{ api_host }}"
+      api_user: "{{ user }}@{{ domain }}"
+      api_password: "{{ api_password | default(omit) }}"
+      api_token_id: "{{ api_token_id | default(omit) }}"
+      api_token_secret: "{{ api_token_secret | default(omit) }}"
+      validate_certs: "{{ validate_certs }}"
+      poolid: "{{ poolid }}"
+      state: absent
+    check_mode: true
+    register: result
+
+  - assert:
+      that:
+        - result is changed
+        - result is success
+
+  - name: Delete non-existing pool should do nothing
+    proxmox_pool:
+      api_host: "{{ api_host }}"
+      api_user: "{{ user }}@{{ domain }}"
+      api_password: "{{ api_password | default(omit) }}"
+      api_token_id: "{{ api_token_id | default(omit) }}"
+      api_token_secret: "{{ api_token_secret | default(omit) }}"
+      validate_certs: "{{ validate_certs }}"
+      poolid: "non-existing-poolid"
+      state: absent
+    register: result
+
+  - assert:
+      that:
+        - result is not changed
+        - result is success
+
+  - name: Deletion of non-empty pool fails
+    block:
+      - name: Add storage into pool
+        proxmox_pool_member:
+          api_host: "{{ api_host }}"
+          api_user: "{{ user }}@{{ domain }}"
+          api_password: "{{ api_password | default(omit) }}"
+          api_token_id: "{{ api_token_id | default(omit) }}"
+          api_token_secret: "{{ api_token_secret | default(omit) }}"
+          validate_certs: "{{ validate_certs }}"
+          poolid: "{{ poolid }}"
+          member: "{{ member }}"
+          type: "{{ member_type }}"
+        diff: true
+        register: result
+
+      - assert:
+          that:
+            - result is changed
+            - result is success
+            - "'{{ member }}' in result.diff.after.members"
+
+      - name: Add non-existing storage into pool should fail
+        proxmox_pool_member:
+          api_host: "{{ api_host }}"
+          api_user: "{{ user }}@{{ domain }}"
+          api_password: "{{ api_password | default(omit) }}"
+          api_token_id: "{{ api_token_id | default(omit) }}"
+          api_token_secret: "{{ api_token_secret | default(omit) }}"
+          validate_certs: "{{ validate_certs }}"
+          poolid: "{{ poolid }}"
+          member: "non-existing-storage"
+          type: "{{ member_type }}"
+        ignore_errors: true
+        register: result
+
+      - assert:
+          that:
+            - result is failed
+            - "'Storage non-existing-storage doesn\\'t exist in the cluster' in result.msg"
+
+      - name: Delete non-empty pool
+        proxmox_pool:
+          api_host: "{{ api_host }}"
+          api_user: "{{ user }}@{{ domain }}"
+          api_password: "{{ api_password | default(omit) }}"
+          api_token_id: "{{ api_token_id | default(omit) }}"
+          api_token_secret: "{{ api_token_secret | default(omit) }}"
+          validate_certs: "{{ validate_certs }}"
+          poolid: "{{ poolid }}"
+          state: absent
+        ignore_errors: true
+        register: result
+
+      - assert:
+          that:
+            - result is failed
+            - "'Please remove members from pool first.' in result.msg"
+
+      - name: Delete storage from the pool
+        proxmox_pool_member:
+          api_host: "{{ api_host }}"
+          api_user: "{{ user }}@{{ domain }}"
+          api_password: "{{ api_password | default(omit) }}"
+          api_token_id: "{{ api_token_id | default(omit) }}"
+          api_token_secret: "{{ api_token_secret | default(omit) }}"
+          validate_certs: "{{ validate_certs }}"
+          poolid: "{{ poolid }}"
+          member: "{{ member }}"
+          type: "{{ member_type }}"
+          state: absent
+        register: result
+
+      - assert:
+          that:
+            - result is success
+            - result is changed
+
+    rescue:
+      - name: Delete storage from the pool if it is added
+        proxmox_pool_member:
+          api_host: "{{ api_host }}"
+          api_user: "{{ user }}@{{ domain }}"
+          api_password: "{{ api_password | default(omit) }}"
+          api_token_id: "{{ api_token_id | default(omit) }}"
+          api_token_secret: "{{ api_token_secret | default(omit) }}"
+          validate_certs: "{{ validate_certs }}"
+          poolid: "{{ poolid }}"
+          member: "{{ member }}"
+          type: "{{ member_type }}"
+          state: absent
+        ignore_errors: true
+
+  - name: Delete pool
+    proxmox_pool:
+      api_host: "{{ api_host }}"
+      api_user: "{{ user }}@{{ domain }}"
+      api_password: "{{ api_password | default(omit) }}"
+      api_token_id: "{{ api_token_id | default(omit) }}"
+      api_token_secret: "{{ api_token_secret | default(omit) }}"
+      validate_certs: "{{ validate_certs }}"
+      poolid: "{{ poolid }}"
+      state: absent
+    register: result
+
+  - assert:
+      that:
+        - result is changed
+        - result is success
+        - result.poolid == "{{ poolid }}"
+
+  rescue:
+    - name: Delete test pool if it is created
+      proxmox_pool:
+        api_host: "{{ api_host }}"
+        api_user: "{{ user }}@{{ domain }}"
+        api_password: "{{ api_password | default(omit) }}"
+        api_token_id: "{{ api_token_id | default(omit) }}"
+        api_token_secret: "{{ api_token_secret | default(omit) }}"
+        validate_certs: "{{ validate_certs }}"
+        poolid: "{{ poolid }}"
+        state: absent
+      ignore_errors: true