pmem: Add namespace and namespace_append options (#4225) (#4239)

* pmem: Add namespace and namespace_append options

- namespace: Configure the namespace of PMem. PMem should be configured
  by appdirect/memmode, or socket option in advance.
- namespace_append: Enables to append the new namespaces.

* Add changelog fragment entry

* Update the changelog fragment

* Update changelog fragment entry

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update to use human_to_bytes

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update to fix the description of namespace_append

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update to release v4.5.0

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update to fix the typo in the description of namespace_append

Co-authored-by: Felix Fontein <felix@fontein.de>

Co-authored-by: Felix Fontein <felix@fontein.de>
(cherry picked from commit 5841935e37)

Co-authored-by: mizumm <26898888+mizumm@users.noreply.github.com>

plugins/modules/storage/pmem/pmem.py

@@ -76,6 +76,39 @@ options:
         description:
           - Percentage of the capacity to reserve (C(0)-C(100)) within the socket ID.
         type: int
+  namespace:
+    description:
+     - This enables to set the configuration for the namespace of the PMem.
+    type: list
+    elements: dict
+    suboptions:
+      mode:
+        description:
+         - The mode of namespace. The detail of the mode is in the man page of ndctl-create-namespace.
+        type: str
+        required: true
+        choices: ['raw', 'sector', 'fsdax', 'devdax']
+      type:
+        description:
+         - The type of namespace. The detail of the type is in the man page of ndctl-create-namespace.
+        type: str
+        required: false
+        choices: ['pmem', 'blk']
+      size:
+        description:
+          - The size of namespace. This option supports the suffixes C(k) or C(K) or C(KB) for KiB,
+            C(m) or C(M) or C(MB) for MiB, C(g) or C(G) or C(GB) for GiB and C(t) or C(T) or C(TB) for TiB.
+          - This option is required if multiple namespaces are configured.
+          - If this option is not set, all of the avaiable space of a region is configured.
+        type: str
+        required: false
+  namespace_append:
+    description:
+     - Enable to append the new namespaces to the system.
+     - The default is C(false) so the all existing namespaces not listed in I(namespace) are removed.
+    type: bool
+    default: false
+    required: false
 '''
 
 RETURN = r'''
@@ -88,6 +121,7 @@ result:
     description:
      - Shows the value of AppDirect, Memory Mode and Reserved size in bytes.
      - If I(socket) argument is provided, shows the values in each socket with C(socket) which contains the socket ID.
+     - If I(namespace) argument is provided, shows the detail of each namespace.
     returned: success
     type: list
     elements: dict
@@ -104,6 +138,9 @@ result:
         socket:
           description: The socket ID to be configured.
           type: int
+        namespace:
+          description: The list of the detail of namespace.
+          type: list
     sample: [
                 {
                     "appdirect": 111669149696,
@@ -150,12 +187,22 @@ EXAMPLES = r'''
         appdirect: 10
         memorymode: 80
         reserved: 10
+
+- name: Configure the two namespaces.
+  community.general.pmem:
+    namespace:
+      - size: 1GB
+        type: pmem
+        mode: raw
+      - size: 320MB
+        type: pmem
+        mode: sector
 '''
 
 import json
 import re
 import traceback
-from ansible.module_utils.basic import AnsibleModule, missing_required_lib
+from ansible.module_utils.basic import AnsibleModule, missing_required_lib, human_to_bytes
 
 try:
     import xmltodict
@@ -184,16 +231,31 @@ class PersistentMemory(object):
                         reserved=dict(type='int'),
                     ),
                 ),
+                namespace=dict(
+                    type='list', elements='dict',
+                    options=dict(
+                        mode=dict(required=True, type='str', choices=['raw', 'sector', 'fsdax', 'devdax']),
+                        type=dict(type='str', choices=['pmem', 'blk']),
+                        size=dict(type='str'),
+                    ),
+                ),
+                namespace_append=dict(type='bool', default=False),
             ),
             required_together=(
                 ['appdirect', 'memorymode'],
             ),
             required_one_of=(
-                ['appdirect', 'memorymode', 'socket'],
+                ['appdirect', 'memorymode', 'socket', 'namespace'],
             ),
             mutually_exclusive=(
                 ['appdirect', 'socket'],
                 ['memorymode', 'socket'],
+                ['appdirect', 'namespace'],
+                ['memorymode', 'namespace'],
+                ['socket', 'namespace'],
+                ['appdirect', 'namespace_append'],
+                ['memorymode', 'namespace_append'],
+                ['socket', 'namespace_append'],
             ),
         )
 
@@ -210,6 +272,8 @@ class PersistentMemory(object):
         self.memmode = module.params['memorymode']
         self.reserved = module.params['reserved']
         self.socket = module.params['socket']
+        self.namespace = module.params['namespace']
+        self.namespace_append = module.params['namespace_append']
 
         self.module = module
         self.changed = False
@@ -248,7 +312,73 @@ class PersistentMemory(object):
         command = ['show', '-system', '-capabilities']
         return self.pmem_run_ipmctl(command)
 
+    def pmem_get_region_align_size(self, region):
+        aligns = []
+        for rg in region:
+            if rg['align'] not in aligns:
+                aligns.append(rg['align'])
+
+        return aligns
+
+    def pmem_get_available_region_size(self, region):
+        available_size = []
+        for rg in region:
+            available_size.append(rg['available_size'])
+
+        return available_size
+
+    def pmem_get_available_region_type(self, region):
+        types = []
+        for rg in region:
+            if rg['type'] not in types:
+                types.append(rg['type'])
+
+        return types
+
     def pmem_argument_check(self):
+        def namespace_check(self):
+            command = ['list', '-R']
+            out = self.pmem_run_ndctl(command)
+            if not out:
+                return 'Available region(s) is not in this system.'
+            region = json.loads(out)
+
+            aligns = self.pmem_get_region_align_size(region)
+            if len(aligns) != 1:
+                return 'Not supported the regions whose alignment size is different.'
+
+            available_size = self.pmem_get_available_region_size(region)
+            types = self.pmem_get_available_region_type(region)
+            for ns in self.namespace:
+                if ns['size']:
+                    try:
+                        size_byte = human_to_bytes(ns['size'])
+                    except ValueError:
+                        return 'The format of size: NNN TB|GB|MB|KB|T|G|M|K|B'
+
+                    if size_byte % aligns[0] != 0:
+                        return 'size: %s should be align with %d' % (ns['size'], aligns[0])
+
+                    is_space_enough = False
+                    for i, avail in enumerate(available_size):
+                        if avail > size_byte:
+                            available_size[i] -= size_byte
+                            is_space_enough = True
+                            break
+
+                    if is_space_enough is False:
+                        return 'There is not available region for size: %s' % ns['size']
+
+                    ns['size_byte'] = size_byte
+
+                elif len(self.namespace) != 1:
+                    return 'size option is required to configure multiple namespaces'
+
+                if ns['type'] not in types:
+                    return 'type %s is not supported in this system. Supported type: %s' % (ns['type'], types)
+
+            return None
+
         def percent_check(self, appdirect, memmode, reserved=None):
             if appdirect is None or (appdirect < 0 or appdirect > 100):
                 return 'appdirect percent should be from 0 to 100.'
@@ -278,7 +408,9 @@ class PersistentMemory(object):
 
             return None
 
-        if self.socket is None:
+        if self.namespace:
+            return namespace_check(self)
+        elif self.socket is None:
             return percent_check(self, self.appdirect, self.memmode, self.reserved)
         else:
             ret = socket_id_check(self)
@@ -319,8 +451,10 @@ class PersistentMemory(object):
         self.pmem_run_ipmctl(command)
 
     def pmem_init_env(self):
-        self.pmem_remove_namespaces()
+        if self.namespace is None or (self.namespace and self.namespace_append is False):
-        self.pmem_delete_goal()
+            self.pmem_remove_namespaces()
+        if self.namespace is None:
+            self.pmem_delete_goal()
 
     def pmem_get_capacity(self, skt=None):
         command = ['show', '-d', 'Capacity', '-u', 'B', '-o', 'nvmxml', '-dimm']
@@ -431,6 +565,17 @@ class PersistentMemory(object):
 
         return reboot_required, ret, ''
 
+    def pmem_config_namespaces(self, namespace):
+        command = ['create-namespace', '-m', namespace['mode']]
+        if namespace['type']:
+            command += ['-t', namespace['type']]
+        if 'size_byte' in namespace:
+            command += ['-s', namespace['size_byte']]
+
+        self.pmem_run_ndctl(command)
+
+        return None
+
 
 def main():
 
@@ -445,7 +590,17 @@ def main():
     pmem.pmem_init_env()
     pmem.changed = True
 
-    if pmem.socket is None:
+    if pmem.namespace:
+        for ns in pmem.namespace:
+            pmem.pmem_config_namespaces(ns)
+
+        command = ['list', '-N']
+        out = pmem.pmem_run_ndctl(command)
+        all_ns = json.loads(out)
+
+        pmem.result = all_ns
+        reboot_required = False
+    elif pmem.socket is None:
         reboot_required, ret, errmsg = pmem.pmem_create_memory_allocation()
         if errmsg:
             pmem.module.fail_json(msg=errmsg)

tests/unit/plugins/modules/storage/pmem/test_pmem.py

@@ -6,6 +6,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import pytest
+import json
 
 pytest.importorskip('xmltodict')
 
@@ -165,6 +166,102 @@ show_skt = """<?xml version="1.0"?>
  </Socket>
 </SocketList>"""
 
+# ndctl_region: the mock return value of pmem_run_command with:
+# ndctl list -R
+ndctl_region = """[
+  {
+    "dev":"region1",
+    "size":51539607552,
+    "align":16777216,
+    "available_size":50465865728,
+    "max_available_extent":50465865728,
+    "type":"pmem",
+    "iset_id":4694373484956518536,
+    "persistence_domain":"memory_controller"
+  },
+  {
+    "dev":"region0",
+    "size":51539607552,
+    "align":16777216,
+    "available_size":51539607552,
+    "max_available_extent":51539607552,
+    "type":"pmem",
+    "iset_id":4588538894081362056,
+    "persistence_domain":"memory_controller"
+  }
+]"""
+
+# ndctl_region_empty: the mock return value of pmem_run_command with:
+# ndctl list -R
+ndctl_region_empty = ""
+
+# ndctl_without_size: the mock return value of pmem_run_command with:
+# ndctl create-namespace -t pmem -m sector
+ndctl_create_without_size = """{
+  "dev":"namespace1.0",
+  "mode":"sector",
+  "size":"47.95 GiB (51.49 GB)",
+  "uuid":"1aca23a5-941c-4f4a-9d88-e531f0b5a27e",
+  "sector_size":4096,
+  "blockdev":"pmem1s"
+}"""
+
+# ndctl_list_N: the mock return value of pmem_run_command with:
+# ndctl list -N
+ndctl_list_N = """[
+  {
+    "dev":"namespace1.0",
+    "mode":"sector",
+    "size":51488243712,
+    "uuid":"1aca23a5-941c-4f4a-9d88-e531f0b5a27e",
+    "sector_size":4096,
+    "blockdev":"pmem1s"
+  }
+]"""
+
+# ndctl_result_1G: the mock return value of pmem_run_command with:
+# ndctl create-namespace -t pmem -m sector -s 1073741824
+ndctl_create_1G = """{
+  "dev":"namespace0.0",
+  "mode":"sector",
+  "size":"1021.97 MiB (1071.62 MB)",
+  "uuid":"5ba4e51b-3028-4b06-8495-b6834867a9af",
+  "sector_size":4096,
+  "blockdev":"pmem0s"
+}"""
+
+# ndctl_result_640M: the mock return value of pmem_run_command with:
+# ndctl create-namespace -t pmem -m raw -s 671088640
+ndctl_create_640M = """{
+  "dev":"namespace1.0",
+  "mode":"raw",
+  "size":"640.00 MiB (671.09 MB)",
+  "uuid":"5ac1f81d-86e6-4f07-9460-8c4d37027f7a",
+  "sector_size":512,
+  "blockdev":"pmem1"
+}"""
+
+# ndctl_list_N_tow_namespaces: the mock return value of pmem_run_command with:
+# ndctl list -N
+ndctl_list_N_two_namespaces = """[
+  {
+    "dev":"namespace1.0",
+    "mode":"sector",
+    "size":1071616000,
+    "uuid":"afcf050d-3a8b-4f48-88a5-16d7c40ab2d8",
+    "sector_size":4096,
+    "blockdev":"pmem1s"
+  },
+  {
+    "dev":"namespace1.1",
+    "mode":"raw",
+    "size":671088640,
+    "uuid":"fb704339-729b-4cc7-b260-079f2633d84f",
+    "sector_size":512,
+    "blockdev":"pmem1.1"
+  }
+]"""
+
 
 class TestPmem(ModuleTestCase):
     def setUp(self):
@@ -210,6 +307,17 @@ class TestPmem(ModuleTestCase):
             if socket:
                 self.assertAlmostEqual(test_result[i]['socket'], i)
 
+    def result_check_ns(self, result, namespace):
+        self.assertTrue(result.exception.args[0]['changed'])
+        self.assertFalse(result.exception.args[0]['reboot_required'])
+
+        test_result = result.exception.args[0]['result']
+        expected = json.loads(namespace)
+
+        for i, notuse in enumerate(test_result):
+            self.assertEqual(test_result[i]['dev'], expected[i]['dev'])
+            self.assertEqual(test_result[i]['size'], expected[i]['size'])
+
     def test_fail_when_required_args_missing(self):
         with self.assertRaises(AnsibleFailJson):
             set_module_args({})
@@ -409,3 +517,190 @@ class TestPmem(ModuleTestCase):
                 pmem_module.main()
             self.result_check(
                 result, True, [12884901888, 12884901888], [94489280512, 94489280512], [164115382272, 164115382272])
+
+    def test_fail_when_namespace_without_mode(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'size': '1GB',
+                        'type': 'pmem',
+                    },
+                    {
+                        'size': '2GB',
+                        'type': 'blk',
+                    },
+                ],
+            })
+            pmem_module.main()
+
+    def test_fail_when_region_is_empty(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'size': '1GB',
+                        'type': 'pmem',
+                        'mode': 'sector',
+                    },
+                ],
+            })
+            with patch(
+                    'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                    side_effect=[ndctl_region_empty]):
+                pmem_module.main()
+
+    def test_fail_when_namespace_invalid_size(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'size': '1XXX',
+                        'type': 'pmem',
+                        'mode': 'sector',
+                    },
+                ],
+            })
+            with patch(
+                    'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                    side_effect=[ndctl_region]):
+                pmem_module.main()
+
+    def test_fail_when_size_is_invalid_alignment(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'size': '400MB',
+                        'type': 'pmem',
+                        'mode': 'sector'
+                    },
+                    {
+                        'size': '500MB',
+                        'type': 'pmem',
+                        'mode': 'sector'
+                    },
+                ],
+            })
+            with patch(
+                    'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                    side_effect=[ndctl_region]):
+                pmem_module.main()
+
+    def test_fail_when_blk_is_unsupported_type(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'size': '4GB',
+                        'type': 'pmem',
+                        'mode': 'sector'
+                    },
+                    {
+                        'size': '5GB',
+                        'type': 'blk',
+                        'mode': 'sector'
+                    },
+                ],
+            })
+            with patch(
+                    'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                    side_effect=[ndctl_region]):
+                pmem_module.main()
+
+    def test_fail_when_size_isnot_set_to_multiple_namespaces(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'type': 'pmem',
+                        'mode': 'sector'
+                    },
+                    {
+                        'size': '500GB',
+                        'type': 'blk',
+                        'mode': 'sector'
+                    },
+                ],
+            })
+            with patch(
+                    'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                    side_effect=[ndctl_region]):
+                pmem_module.main()
+
+    def test_fail_when_size_of_namespace_over_available(self):
+        with self.assertRaises(AnsibleFailJson):
+            set_module_args({
+                'namespace': [
+                    {
+                        'size': '400GB',
+                        'type': 'pmem',
+                        'mode': 'sector'
+                    },
+                    {
+                        'size': '500GB',
+                        'type': 'pmem',
+                        'mode': 'sector'
+                    },
+                ],
+            })
+            with patch(
+                    'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                    side_effect=[ndctl_region]):
+                pmem_module.main()
+
+    def test_when_namespace0_without_size(self):
+        set_module_args({
+            'namespace': [
+                {
+                    'type': 'pmem',
+                    'mode': 'sector'
+                },
+            ],
+        })
+        with patch(
+                'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                side_effect=[ndctl_region, ndctl_create_without_size, ndctl_list_N]):
+            with self.assertRaises(AnsibleExitJson) as result:
+                pmem_module.main()
+            self.result_check_ns(result, ndctl_list_N)
+
+    def test_when_namespace0_with_namespace_append(self):
+        set_module_args({
+            'namespace': [
+                {
+                    'size': '640MB',
+                    'type': 'pmem',
+                    'mode': 'raw'
+                },
+            ],
+            'namespace_append': True,
+        })
+        with patch(
+                'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                side_effect=[ndctl_region, ndctl_create_640M, ndctl_list_N_two_namespaces]):
+            with self.assertRaises(AnsibleExitJson) as result:
+                pmem_module.main()
+            self.result_check_ns(result, ndctl_list_N_two_namespaces)
+
+    def test_when_namespace0_1GiB_pmem_sector_namespace1_640MiB_pmem_raw(self):
+        set_module_args({
+            'namespace': [
+                {
+                    'size': '1GB',
+                    'type': 'pmem',
+                    'mode': 'sector'
+                },
+                {
+                    'size': '640MB',
+                    'type': 'pmem',
+                    'mode': 'raw',
+                },
+            ],
+        })
+        with patch(
+                'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command',
+                side_effect=[ndctl_region, ndctl_create_1G, ndctl_create_640M, ndctl_list_N_two_namespaces]):
+            with self.assertRaises(AnsibleExitJson) as result:
+                pmem_module.main()
+            self.result_check_ns(result, ndctl_list_N_two_namespaces)