From 5841935e3737817b03b57c452951b72216b30572 Mon Sep 17 00:00:00 2001 From: mizumm <26898888+mizumm@users.noreply.github.com> Date: Sun, 20 Feb 2022 16:10:13 -0500 Subject: [PATCH] pmem: Add namespace and namespace_append options (#4225) * pmem: Add namespace and namespace_append options - namespace: Configure the namespace of PMem. PMem should be configured by appdirect/memmode, or socket option in advance. - namespace_append: Enables to append the new namespaces. * Add changelog fragment entry * Update the changelog fragment * Update changelog fragment entry Co-authored-by: Felix Fontein * Update to use human_to_bytes Co-authored-by: Felix Fontein * Update to fix the description of namespace_append Co-authored-by: Felix Fontein * Update to release v4.5.0 Co-authored-by: Felix Fontein * Update to fix the typo in the description of namespace_append Co-authored-by: Felix Fontein Co-authored-by: Felix Fontein --- plugins/modules/storage/pmem/pmem.py | 167 +++++++++- .../plugins/modules/storage/pmem/test_pmem.py | 295 ++++++++++++++++++ 2 files changed, 456 insertions(+), 6 deletions(-) diff --git a/plugins/modules/storage/pmem/pmem.py b/plugins/modules/storage/pmem/pmem.py index 2ad5d66209..b91bab5fad 100644 --- a/plugins/modules/storage/pmem/pmem.py +++ b/plugins/modules/storage/pmem/pmem.py @@ -76,6 +76,39 @@ options: description: - Percentage of the capacity to reserve (C(0)-C(100)) within the socket ID. type: int + namespace: + description: + - This enables to set the configuration for the namespace of the PMem. + type: list + elements: dict + suboptions: + mode: + description: + - The mode of namespace. The detail of the mode is in the man page of ndctl-create-namespace. + type: str + required: true + choices: ['raw', 'sector', 'fsdax', 'devdax'] + type: + description: + - The type of namespace. The detail of the type is in the man page of ndctl-create-namespace. + type: str + required: false + choices: ['pmem', 'blk'] + size: + description: + - The size of namespace. This option supports the suffixes C(k) or C(K) or C(KB) for KiB, + C(m) or C(M) or C(MB) for MiB, C(g) or C(G) or C(GB) for GiB and C(t) or C(T) or C(TB) for TiB. + - This option is required if multiple namespaces are configured. + - If this option is not set, all of the avaiable space of a region is configured. + type: str + required: false + namespace_append: + description: + - Enable to append the new namespaces to the system. + - The default is C(false) so the all existing namespaces not listed in I(namespace) are removed. + type: bool + default: false + required: false ''' RETURN = r''' @@ -88,6 +121,7 @@ result: description: - Shows the value of AppDirect, Memory Mode and Reserved size in bytes. - If I(socket) argument is provided, shows the values in each socket with C(socket) which contains the socket ID. + - If I(namespace) argument is provided, shows the detail of each namespace. returned: success type: list elements: dict @@ -104,6 +138,9 @@ result: socket: description: The socket ID to be configured. type: int + namespace: + description: The list of the detail of namespace. + type: list sample: [ { "appdirect": 111669149696, @@ -150,12 +187,22 @@ EXAMPLES = r''' appdirect: 10 memorymode: 80 reserved: 10 + +- name: Configure the two namespaces. + community.general.pmem: + namespace: + - size: 1GB + type: pmem + mode: raw + - size: 320MB + type: pmem + mode: sector ''' import json import re import traceback -from ansible.module_utils.basic import AnsibleModule, missing_required_lib +from ansible.module_utils.basic import AnsibleModule, missing_required_lib, human_to_bytes try: import xmltodict @@ -184,16 +231,31 @@ class PersistentMemory(object): reserved=dict(type='int'), ), ), + namespace=dict( + type='list', elements='dict', + options=dict( + mode=dict(required=True, type='str', choices=['raw', 'sector', 'fsdax', 'devdax']), + type=dict(type='str', choices=['pmem', 'blk']), + size=dict(type='str'), + ), + ), + namespace_append=dict(type='bool', default=False), ), required_together=( ['appdirect', 'memorymode'], ), required_one_of=( - ['appdirect', 'memorymode', 'socket'], + ['appdirect', 'memorymode', 'socket', 'namespace'], ), mutually_exclusive=( ['appdirect', 'socket'], ['memorymode', 'socket'], + ['appdirect', 'namespace'], + ['memorymode', 'namespace'], + ['socket', 'namespace'], + ['appdirect', 'namespace_append'], + ['memorymode', 'namespace_append'], + ['socket', 'namespace_append'], ), ) @@ -210,6 +272,8 @@ class PersistentMemory(object): self.memmode = module.params['memorymode'] self.reserved = module.params['reserved'] self.socket = module.params['socket'] + self.namespace = module.params['namespace'] + self.namespace_append = module.params['namespace_append'] self.module = module self.changed = False @@ -248,7 +312,73 @@ class PersistentMemory(object): command = ['show', '-system', '-capabilities'] return self.pmem_run_ipmctl(command) + def pmem_get_region_align_size(self, region): + aligns = [] + for rg in region: + if rg['align'] not in aligns: + aligns.append(rg['align']) + + return aligns + + def pmem_get_available_region_size(self, region): + available_size = [] + for rg in region: + available_size.append(rg['available_size']) + + return available_size + + def pmem_get_available_region_type(self, region): + types = [] + for rg in region: + if rg['type'] not in types: + types.append(rg['type']) + + return types + def pmem_argument_check(self): + def namespace_check(self): + command = ['list', '-R'] + out = self.pmem_run_ndctl(command) + if not out: + return 'Available region(s) is not in this system.' + region = json.loads(out) + + aligns = self.pmem_get_region_align_size(region) + if len(aligns) != 1: + return 'Not supported the regions whose alignment size is different.' + + available_size = self.pmem_get_available_region_size(region) + types = self.pmem_get_available_region_type(region) + for ns in self.namespace: + if ns['size']: + try: + size_byte = human_to_bytes(ns['size']) + except ValueError: + return 'The format of size: NNN TB|GB|MB|KB|T|G|M|K|B' + + if size_byte % aligns[0] != 0: + return 'size: %s should be align with %d' % (ns['size'], aligns[0]) + + is_space_enough = False + for i, avail in enumerate(available_size): + if avail > size_byte: + available_size[i] -= size_byte + is_space_enough = True + break + + if is_space_enough is False: + return 'There is not available region for size: %s' % ns['size'] + + ns['size_byte'] = size_byte + + elif len(self.namespace) != 1: + return 'size option is required to configure multiple namespaces' + + if ns['type'] not in types: + return 'type %s is not supported in this system. Supported type: %s' % (ns['type'], types) + + return None + def percent_check(self, appdirect, memmode, reserved=None): if appdirect is None or (appdirect < 0 or appdirect > 100): return 'appdirect percent should be from 0 to 100.' @@ -278,7 +408,9 @@ class PersistentMemory(object): return None - if self.socket is None: + if self.namespace: + return namespace_check(self) + elif self.socket is None: return percent_check(self, self.appdirect, self.memmode, self.reserved) else: ret = socket_id_check(self) @@ -319,8 +451,10 @@ class PersistentMemory(object): self.pmem_run_ipmctl(command) def pmem_init_env(self): - self.pmem_remove_namespaces() - self.pmem_delete_goal() + if self.namespace is None or (self.namespace and self.namespace_append is False): + self.pmem_remove_namespaces() + if self.namespace is None: + self.pmem_delete_goal() def pmem_get_capacity(self, skt=None): command = ['show', '-d', 'Capacity', '-u', 'B', '-o', 'nvmxml', '-dimm'] @@ -431,6 +565,17 @@ class PersistentMemory(object): return reboot_required, ret, '' + def pmem_config_namespaces(self, namespace): + command = ['create-namespace', '-m', namespace['mode']] + if namespace['type']: + command += ['-t', namespace['type']] + if 'size_byte' in namespace: + command += ['-s', namespace['size_byte']] + + self.pmem_run_ndctl(command) + + return None + def main(): @@ -445,7 +590,17 @@ def main(): pmem.pmem_init_env() pmem.changed = True - if pmem.socket is None: + if pmem.namespace: + for ns in pmem.namespace: + pmem.pmem_config_namespaces(ns) + + command = ['list', '-N'] + out = pmem.pmem_run_ndctl(command) + all_ns = json.loads(out) + + pmem.result = all_ns + reboot_required = False + elif pmem.socket is None: reboot_required, ret, errmsg = pmem.pmem_create_memory_allocation() if errmsg: pmem.module.fail_json(msg=errmsg) diff --git a/tests/unit/plugins/modules/storage/pmem/test_pmem.py b/tests/unit/plugins/modules/storage/pmem/test_pmem.py index cfe6dc8b59..b3d072080b 100644 --- a/tests/unit/plugins/modules/storage/pmem/test_pmem.py +++ b/tests/unit/plugins/modules/storage/pmem/test_pmem.py @@ -6,6 +6,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import pytest +import json pytest.importorskip('xmltodict') @@ -165,6 +166,102 @@ show_skt = """ """ +# ndctl_region: the mock return value of pmem_run_command with: +# ndctl list -R +ndctl_region = """[ + { + "dev":"region1", + "size":51539607552, + "align":16777216, + "available_size":50465865728, + "max_available_extent":50465865728, + "type":"pmem", + "iset_id":4694373484956518536, + "persistence_domain":"memory_controller" + }, + { + "dev":"region0", + "size":51539607552, + "align":16777216, + "available_size":51539607552, + "max_available_extent":51539607552, + "type":"pmem", + "iset_id":4588538894081362056, + "persistence_domain":"memory_controller" + } +]""" + +# ndctl_region_empty: the mock return value of pmem_run_command with: +# ndctl list -R +ndctl_region_empty = "" + +# ndctl_without_size: the mock return value of pmem_run_command with: +# ndctl create-namespace -t pmem -m sector +ndctl_create_without_size = """{ + "dev":"namespace1.0", + "mode":"sector", + "size":"47.95 GiB (51.49 GB)", + "uuid":"1aca23a5-941c-4f4a-9d88-e531f0b5a27e", + "sector_size":4096, + "blockdev":"pmem1s" +}""" + +# ndctl_list_N: the mock return value of pmem_run_command with: +# ndctl list -N +ndctl_list_N = """[ + { + "dev":"namespace1.0", + "mode":"sector", + "size":51488243712, + "uuid":"1aca23a5-941c-4f4a-9d88-e531f0b5a27e", + "sector_size":4096, + "blockdev":"pmem1s" + } +]""" + +# ndctl_result_1G: the mock return value of pmem_run_command with: +# ndctl create-namespace -t pmem -m sector -s 1073741824 +ndctl_create_1G = """{ + "dev":"namespace0.0", + "mode":"sector", + "size":"1021.97 MiB (1071.62 MB)", + "uuid":"5ba4e51b-3028-4b06-8495-b6834867a9af", + "sector_size":4096, + "blockdev":"pmem0s" +}""" + +# ndctl_result_640M: the mock return value of pmem_run_command with: +# ndctl create-namespace -t pmem -m raw -s 671088640 +ndctl_create_640M = """{ + "dev":"namespace1.0", + "mode":"raw", + "size":"640.00 MiB (671.09 MB)", + "uuid":"5ac1f81d-86e6-4f07-9460-8c4d37027f7a", + "sector_size":512, + "blockdev":"pmem1" +}""" + +# ndctl_list_N_tow_namespaces: the mock return value of pmem_run_command with: +# ndctl list -N +ndctl_list_N_two_namespaces = """[ + { + "dev":"namespace1.0", + "mode":"sector", + "size":1071616000, + "uuid":"afcf050d-3a8b-4f48-88a5-16d7c40ab2d8", + "sector_size":4096, + "blockdev":"pmem1s" + }, + { + "dev":"namespace1.1", + "mode":"raw", + "size":671088640, + "uuid":"fb704339-729b-4cc7-b260-079f2633d84f", + "sector_size":512, + "blockdev":"pmem1.1" + } +]""" + class TestPmem(ModuleTestCase): def setUp(self): @@ -210,6 +307,17 @@ class TestPmem(ModuleTestCase): if socket: self.assertAlmostEqual(test_result[i]['socket'], i) + def result_check_ns(self, result, namespace): + self.assertTrue(result.exception.args[0]['changed']) + self.assertFalse(result.exception.args[0]['reboot_required']) + + test_result = result.exception.args[0]['result'] + expected = json.loads(namespace) + + for i, notuse in enumerate(test_result): + self.assertEqual(test_result[i]['dev'], expected[i]['dev']) + self.assertEqual(test_result[i]['size'], expected[i]['size']) + def test_fail_when_required_args_missing(self): with self.assertRaises(AnsibleFailJson): set_module_args({}) @@ -409,3 +517,190 @@ class TestPmem(ModuleTestCase): pmem_module.main() self.result_check( result, True, [12884901888, 12884901888], [94489280512, 94489280512], [164115382272, 164115382272]) + + def test_fail_when_namespace_without_mode(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'size': '1GB', + 'type': 'pmem', + }, + { + 'size': '2GB', + 'type': 'blk', + }, + ], + }) + pmem_module.main() + + def test_fail_when_region_is_empty(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'size': '1GB', + 'type': 'pmem', + 'mode': 'sector', + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region_empty]): + pmem_module.main() + + def test_fail_when_namespace_invalid_size(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'size': '1XXX', + 'type': 'pmem', + 'mode': 'sector', + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region]): + pmem_module.main() + + def test_fail_when_size_is_invalid_alignment(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'size': '400MB', + 'type': 'pmem', + 'mode': 'sector' + }, + { + 'size': '500MB', + 'type': 'pmem', + 'mode': 'sector' + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region]): + pmem_module.main() + + def test_fail_when_blk_is_unsupported_type(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'size': '4GB', + 'type': 'pmem', + 'mode': 'sector' + }, + { + 'size': '5GB', + 'type': 'blk', + 'mode': 'sector' + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region]): + pmem_module.main() + + def test_fail_when_size_isnot_set_to_multiple_namespaces(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'type': 'pmem', + 'mode': 'sector' + }, + { + 'size': '500GB', + 'type': 'blk', + 'mode': 'sector' + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region]): + pmem_module.main() + + def test_fail_when_size_of_namespace_over_available(self): + with self.assertRaises(AnsibleFailJson): + set_module_args({ + 'namespace': [ + { + 'size': '400GB', + 'type': 'pmem', + 'mode': 'sector' + }, + { + 'size': '500GB', + 'type': 'pmem', + 'mode': 'sector' + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region]): + pmem_module.main() + + def test_when_namespace0_without_size(self): + set_module_args({ + 'namespace': [ + { + 'type': 'pmem', + 'mode': 'sector' + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region, ndctl_create_without_size, ndctl_list_N]): + with self.assertRaises(AnsibleExitJson) as result: + pmem_module.main() + self.result_check_ns(result, ndctl_list_N) + + def test_when_namespace0_with_namespace_append(self): + set_module_args({ + 'namespace': [ + { + 'size': '640MB', + 'type': 'pmem', + 'mode': 'raw' + }, + ], + 'namespace_append': True, + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region, ndctl_create_640M, ndctl_list_N_two_namespaces]): + with self.assertRaises(AnsibleExitJson) as result: + pmem_module.main() + self.result_check_ns(result, ndctl_list_N_two_namespaces) + + def test_when_namespace0_1GiB_pmem_sector_namespace1_640MiB_pmem_raw(self): + set_module_args({ + 'namespace': [ + { + 'size': '1GB', + 'type': 'pmem', + 'mode': 'sector' + }, + { + 'size': '640MB', + 'type': 'pmem', + 'mode': 'raw', + }, + ], + }) + with patch( + 'ansible_collections.community.general.plugins.modules.storage.pmem.pmem.PersistentMemory.pmem_run_command', + side_effect=[ndctl_region, ndctl_create_1G, ndctl_create_640M, ndctl_list_N_two_namespaces]): + with self.assertRaises(AnsibleExitJson) as result: + pmem_module.main() + self.result_check_ns(result, ndctl_list_N_two_namespaces)