module_helper.py Breakdown (#2393) (#2492)

* break down of module_helper into smaller pieces, keeping compatibility

* removed abc.ABC (py3 only) from code + fixed reference to vars.py

* multiple changes:

- mh.base - moved more functionalities to ModuleHelperBase
- mh.mixins.(cmd, state) - CmdMixin no longer inherits from ModuleHelperBase
- mh.mixins.deps - DependencyMixin now overrides run() method to test dependency
- mh.mixins.vars - created class VarsMixin
- mh.module_helper - moved functions to base class, added VarsMixin
- module_helper - importing AnsibleModule as well, for backward compatibility in test

* removed unnecessary __all__

* make pylint happy

* PR adjustments + bot config + changelog frag

* Update plugins/module_utils/mh/module_helper.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/module_utils/mh/module_helper.py

Co-authored-by: Felix Fontein <felix@fontein.de>

Co-authored-by: Felix Fontein <felix@fontein.de>
(cherry picked from commit d22dd5056e)

Co-authored-by: Alexei Znamensky <103110+russoz@users.noreply.github.com>

.github/BOTMETA.yml

@@ -142,6 +142,9 @@ files:
   $module_utils/memset.py:
     maintainers: glitchcrab
     labels: cloud memset
+  $module_utils/mh/:
+    maintainers: russoz
+    labels: module_helper
   $module_utils/module_helper.py:
     maintainers: russoz
     labels: module_helper

changelogs/fragments/2393-module_helper-breakdown.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - module_helper module utils - break down of the long file into smaller pieces (https://github.com/ansible-collections/community.general/pull/2393).

plugins/module_utils/mh/base.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.community.general.plugins.module_utils.mh.exceptions import ModuleHelperException as _MHE
+from ansible_collections.community.general.plugins.module_utils.mh.deco import module_fails_on_exception
+
+
+class ModuleHelperBase(object):
+    module = None
+    ModuleHelperException = _MHE
+
+    def __init__(self, module=None):
+        self._changed = False
+
+        if module:
+            self.module = module
+
+        if not isinstance(self.module, AnsibleModule):
+            self.module = AnsibleModule(**self.module)
+
+    def __init_module__(self):
+        pass
+
+    def __run__(self):
+        raise NotImplementedError()
+
+    def __quit_module__(self):
+        pass
+
+    @property
+    def changed(self):
+        return self._changed
+
+    @changed.setter
+    def changed(self, value):
+        self._changed = value
+
+    def has_changed(self):
+        raise NotImplementedError()
+
+    @property
+    def output(self):
+        raise NotImplementedError()
+
+    @module_fails_on_exception
+    def run(self):
+        self.__init_module__()
+        self.__run__()
+        self.__quit_module__()
+        self.module.exit_json(changed=self.has_changed(), **self.output)

plugins/module_utils/mh/deco.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import traceback
+from functools import wraps
+
+from ansible_collections.community.general.plugins.module_utils.mh.exceptions import ModuleHelperException
+
+
+def cause_changes(on_success=None, on_failure=None):
+
+    def deco(func):
+        if on_success is None and on_failure is None:
+            return func
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                self = args[0]
+                func(*args, **kwargs)
+                if on_success is not None:
+                    self.changed = on_success
+            except Exception:
+                if on_failure is not None:
+                    self.changed = on_failure
+                raise
+
+        return wrapper
+
+    return deco
+
+
+def module_fails_on_exception(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            func(self, *args, **kwargs)
+        except SystemExit:
+            raise
+        except ModuleHelperException as e:
+            if e.update_output:
+                self.update_output(e.update_output)
+            self.module.fail_json(msg=e.msg, exception=traceback.format_exc(),
+                                  output=self.output, vars=self.vars.output(), **self.output)
+        except Exception as e:
+            msg = "Module failed with exception: {0}".format(str(e).strip())
+            self.module.fail_json(msg=msg, exception=traceback.format_exc(),
+                                  output=self.output, vars=self.vars.output(), **self.output)
+    return wrapper

plugins/module_utils/mh/exceptions.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+class ModuleHelperException(Exception):
+    @staticmethod
+    def _get_remove(key, kwargs):
+        if key in kwargs:
+            result = kwargs[key]
+            del kwargs[key]
+            return result
+        return None
+
+    def __init__(self, *args, **kwargs):
+        self.msg = self._get_remove('msg', kwargs) or "Module failed with exception: {0}".format(self)
+        self.update_output = self._get_remove('update_output', kwargs) or {}
+        super(ModuleHelperException, self).__init__(*args)

plugins/module_utils/mh/mixins/cmd.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from functools import partial
+
+
+class ArgFormat(object):
+    """
+    Argument formatter for use as a command line parameter. Used in CmdMixin.
+    """
+    BOOLEAN = 0
+    PRINTF = 1
+    FORMAT = 2
+
+    @staticmethod
+    def stars_deco(num):
+        if num == 1:
+            def deco(f):
+                return lambda v: f(*v)
+            return deco
+        elif num == 2:
+            def deco(f):
+                return lambda v: f(**v)
+            return deco
+
+        return lambda f: f
+
+    def __init__(self, name, fmt=None, style=FORMAT, stars=0):
+        """
+        Creates a CLI-formatter for one specific argument. The argument may be a module parameter or just a named parameter for
+        the CLI command execution.
+        :param name: Name of the argument to be formatted
+        :param fmt: Either a str to be formatted (using or not printf-style) or a callable that does that
+        :param style: Whether arg_format (as str) should use printf-style formatting.
+                             Ignored if arg_format is None or not a str (should be callable).
+        :param stars: A int with 0, 1 or 2 value, indicating to formatting the value as: value, *value or **value
+        """
+        def printf_fmt(_fmt, v):
+            try:
+                return [_fmt % v]
+            except TypeError as e:
+                if e.args[0] != 'not all arguments converted during string formatting':
+                    raise
+                return [_fmt]
+
+        _fmts = {
+            ArgFormat.BOOLEAN: lambda _fmt, v: ([_fmt] if bool(v) else []),
+            ArgFormat.PRINTF: printf_fmt,
+            ArgFormat.FORMAT: lambda _fmt, v: [_fmt.format(v)],
+        }
+
+        self.name = name
+        self.stars = stars
+
+        if fmt is None:
+            fmt = "{0}"
+            style = ArgFormat.FORMAT
+
+        if isinstance(fmt, str):
+            func = _fmts[style]
+            self.arg_format = partial(func, fmt)
+        elif isinstance(fmt, list) or isinstance(fmt, tuple):
+            self.arg_format = lambda v: [_fmts[style](f, v)[0] for f in fmt]
+        elif hasattr(fmt, '__call__'):
+            self.arg_format = fmt
+        else:
+            raise TypeError('Parameter fmt must be either: a string, a list/tuple of '
+                            'strings or a function: type={0}, value={1}'.format(type(fmt), fmt))
+
+        if stars:
+            self.arg_format = (self.stars_deco(stars))(self.arg_format)
+
+    def to_text(self, value):
+        if value is None:
+            return []
+        func = self.arg_format
+        return [str(p) for p in func(value)]
+
+
+class CmdMixin(object):
+    """
+    Mixin for mapping module options to running a CLI command with its arguments.
+    """
+    command = None
+    command_args_formats = {}
+    run_command_fixed_options = {}
+    check_rc = False
+    force_lang = "C"
+
+    @property
+    def module_formats(self):
+        result = {}
+        for param in self.module.params.keys():
+            result[param] = ArgFormat(param)
+        return result
+
+    @property
+    def custom_formats(self):
+        result = {}
+        for param, fmt_spec in self.command_args_formats.items():
+            result[param] = ArgFormat(param, **fmt_spec)
+        return result
+
+    def _calculate_args(self, extra_params=None, params=None):
+        def add_arg_formatted_param(_cmd_args, arg_format, _value):
+            args = list(arg_format.to_text(_value))
+            return _cmd_args + args
+
+        def find_format(_param):
+            return self.custom_formats.get(_param, self.module_formats.get(_param))
+
+        extra_params = extra_params or dict()
+        cmd_args = list([self.command]) if isinstance(self.command, str) else list(self.command)
+        try:
+            cmd_args[0] = self.module.get_bin_path(cmd_args[0], required=True)
+        except ValueError:
+            pass
+        param_list = params if params else self.module.params.keys()
+
+        for param in param_list:
+            if isinstance(param, dict):
+                if len(param) != 1:
+                    raise self.ModuleHelperException("run_command parameter as a dict must "
+                                                     "contain only one key: {0}".format(param))
+                _param = list(param.keys())[0]
+                fmt = find_format(_param)
+                value = param[_param]
+            elif isinstance(param, str):
+                if param in self.module.argument_spec:
+                    fmt = find_format(param)
+                    value = self.module.params[param]
+                elif param in extra_params:
+                    fmt = find_format(param)
+                    value = extra_params[param]
+                else:
+                    self.module.deprecate("Cannot determine value for parameter: {0}. "
+                                          "From version 4.0.0 onwards this will generate an exception".format(param),
+                                          version="4.0.0", collection_name="community.general")
+                    continue
+
+            else:
+                raise self.ModuleHelperException("run_command parameter must be either a str or a dict: {0}".format(param))
+            cmd_args = add_arg_formatted_param(cmd_args, fmt, value)
+
+        return cmd_args
+
+    def process_command_output(self, rc, out, err):
+        return rc, out, err
+
+    def run_command(self, extra_params=None, params=None, *args, **kwargs):
+        self.vars.cmd_args = self._calculate_args(extra_params, params)
+        options = dict(self.run_command_fixed_options)
+        env_update = dict(options.get('environ_update', {}))
+        options['check_rc'] = options.get('check_rc', self.check_rc)
+        if self.force_lang:
+            env_update.update({'LANGUAGE': self.force_lang})
+            self.update_output(force_lang=self.force_lang)
+            options['environ_update'] = env_update
+        options.update(kwargs)
+        rc, out, err = self.module.run_command(self.vars.cmd_args, *args, **options)
+        self.update_output(rc=rc, stdout=out, stderr=err)
+        return self.process_command_output(rc, out, err)

plugins/module_utils/mh/mixins/deps.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import traceback
+
+from ansible_collections.community.general.plugins.module_utils.mh.base import ModuleHelperBase
+from ansible_collections.community.general.plugins.module_utils.mh.deco import module_fails_on_exception
+
+
+class DependencyCtxMgr(object):
+    def __init__(self, name, msg=None):
+        self.name = name
+        self.msg = msg
+        self.has_it = False
+        self.exc_type = None
+        self.exc_val = None
+        self.exc_tb = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.has_it = exc_type is None
+        self.exc_type = exc_type
+        self.exc_val = exc_val
+        self.exc_tb = exc_tb
+        return not self.has_it
+
+    @property
+    def text(self):
+        return self.msg or str(self.exc_val)
+
+
+class DependencyMixin(ModuleHelperBase):
+    _dependencies = []
+
+    @classmethod
+    def dependency(cls, name, msg):
+        cls._dependencies.append(DependencyCtxMgr(name, msg))
+        return cls._dependencies[-1]
+
+    def fail_on_missing_deps(self):
+        for d in self._dependencies:
+            if not d.has_it:
+                self.module.fail_json(changed=False,
+                                      exception="\n".join(traceback.format_exception(d.exc_type, d.exc_val, d.exc_tb)),
+                                      msg=d.text,
+                                      **self.output)
+
+    @module_fails_on_exception
+    def run(self):
+        self.fail_on_missing_deps()
+        super(DependencyMixin, self).run()

plugins/module_utils/mh/mixins/state.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+class StateMixin(object):
+    state_param = 'state'
+    default_state = None
+
+    def _state(self):
+        state = self.module.params.get(self.state_param)
+        return self.default_state if state is None else state
+
+    def _method(self, state):
+        return "{0}_{1}".format(self.state_param, state)
+
+    def __run__(self):
+        state = self._state()
+        self.vars.state = state
+
+        # resolve aliases
+        if state not in self.module.params:
+            aliased = [name for name, param in self.module.argument_spec.items() if state in param.get('aliases', [])]
+            if aliased:
+                state = aliased[0]
+                self.vars.effective_state = state
+
+        method = self._method(state)
+        if not hasattr(self, method):
+            return self.__state_fallback__()
+        func = getattr(self, method)
+        return func()
+
+    def __state_fallback__(self):
+        raise ValueError("Cannot find method: {0}".format(self._method(self._state())))

plugins/module_utils/mh/mixins/vars.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+class VarMeta(object):
+    NOTHING = object()
+
+    def __init__(self, diff=False, output=True, change=None, fact=False):
+        self.init = False
+        self.initial_value = None
+        self.value = None
+
+        self.diff = diff
+        self.change = diff if change is None else change
+        self.output = output
+        self.fact = fact
+
+    def set(self, diff=None, output=None, change=None, fact=None, initial_value=NOTHING):
+        if diff is not None:
+            self.diff = diff
+        if output is not None:
+            self.output = output
+        if change is not None:
+            self.change = change
+        if fact is not None:
+            self.fact = fact
+        if initial_value is not self.NOTHING:
+            self.initial_value = initial_value
+
+    def set_value(self, value):
+        if not self.init:
+            self.initial_value = value
+            self.init = True
+        self.value = value
+        return self
+
+    @property
+    def has_changed(self):
+        return self.change and (self.initial_value != self.value)
+
+    @property
+    def diff_result(self):
+        return None if not (self.diff and self.has_changed) else {
+            'before': self.initial_value,
+            'after': self.value,
+        }
+
+    def __str__(self):
+        return "<VarMeta: value={0}, initial={1}, diff={2}, output={3}, change={4}>".format(
+            self.value, self.initial_value, self.diff, self.output, self.change
+        )
+
+
+class VarDict(object):
+    def __init__(self):
+        self._data = dict()
+        self._meta = dict()
+
+    def __getitem__(self, item):
+        return self._data[item]
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def __getattr__(self, item):
+        try:
+            return self._data[item]
+        except KeyError:
+            return getattr(self._data, item)
+
+    def __setattr__(self, key, value):
+        if key in ('_data', '_meta'):
+            super(VarDict, self).__setattr__(key, value)
+        else:
+            self.set(key, value)
+
+    def meta(self, name):
+        return self._meta[name]
+
+    def set_meta(self, name, **kwargs):
+        self.meta(name).set(**kwargs)
+
+    def set(self, name, value, **kwargs):
+        if name in ('_data', '_meta'):
+            raise ValueError("Names _data and _meta are reserved for use by ModuleHelper")
+        self._data[name] = value
+        if name in self._meta:
+            meta = self.meta(name)
+        else:
+            meta = VarMeta(**kwargs)
+        meta.set_value(value)
+        self._meta[name] = meta
+
+    def output(self):
+        return dict((k, v) for k, v in self._data.items() if self.meta(k).output)
+
+    def diff(self):
+        diff_results = [(k, self.meta(k).diff_result) for k in self._data]
+        diff_results = [dr for dr in diff_results if dr[1] is not None]
+        if diff_results:
+            before = dict((dr[0], dr[1]['before']) for dr in diff_results)
+            after = dict((dr[0], dr[1]['after']) for dr in diff_results)
+            return {'before': before, 'after': after}
+        return None
+
+    def facts(self):
+        facts_result = dict((k, v) for k, v in self._data.items() if self._meta[k].fact)
+        return facts_result if facts_result else None
+
+    def change_vars(self):
+        return [v for v in self._data if self.meta(v).change]
+
+    def has_changed(self, v):
+        return self._meta[v].has_changed
+
+
+class VarsMixin(object):
+
+    def __init__(self, module=None):
+        self.vars = VarDict()
+        super(VarsMixin, self).__init__(module)
+
+    def update_vars(self, meta=None, **kwargs):
+        if meta is None:
+            meta = {}
+        for k, v in kwargs.items():
+            self.vars.set(k, v, **meta)

plugins/module_utils/mh/module_helper.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Alexei Znamensky <russoz@gmail.com>
+# Copyright: (c) 2020, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.module_utils.common.dict_transformations import dict_merge
+
+from ansible_collections.community.general.plugins.module_utils.mh.base import ModuleHelperBase, AnsibleModule
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.cmd import CmdMixin
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.state import StateMixin
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.deps import DependencyMixin
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.vars import VarsMixin, VarDict as _VD
+
+
+class ModuleHelper(VarsMixin, DependencyMixin, ModuleHelperBase):
+    _output_conflict_list = ('msg', 'exception', 'output', 'vars', 'changed')
+    facts_name = None
+    output_params = ()
+    diff_params = ()
+    change_params = ()
+    facts_params = ()
+
+    VarDict = _VD  # for backward compatibility, will be deprecated at some point
+
+    def __init__(self, module=None):
+        super(ModuleHelper, self).__init__(module)
+        for name, value in self.module.params.items():
+            self.vars.set(
+                name, value,
+                diff=name in self.diff_params,
+                output=name in self.output_params,
+                change=None if not self.change_params else name in self.change_params,
+                fact=name in self.facts_params,
+            )
+
+    def update_output(self, **kwargs):
+        self.update_vars(meta={"output": True}, **kwargs)
+
+    def update_facts(self, **kwargs):
+        self.update_vars(meta={"fact": True}, **kwargs)
+
+    def _vars_changed(self):
+        return any(self.vars.has_changed(v) for v in self.vars.change_vars())
+
+    def has_changed(self):
+        return self.changed or self._vars_changed()
+
+    @property
+    def output(self):
+        result = dict(self.vars.output())
+        if self.facts_name:
+            facts = self.vars.facts()
+            if facts is not None:
+                result['ansible_facts'] = {self.facts_name: facts}
+        if self.module._diff:
+            diff = result.get('diff', {})
+            vars_diff = self.vars.diff() or {}
+            result['diff'] = dict_merge(dict(diff), vars_diff)
+
+        for varname in result:
+            if varname in self._output_conflict_list:
+                result["_" + varname] = result[varname]
+                del result[varname]
+        return result
+
+
+class StateModuleHelper(StateMixin, ModuleHelper):
+    pass
+
+
+class CmdModuleHelper(CmdMixin, ModuleHelper):
+    pass
+
+
+class CmdStateModuleHelper(CmdMixin, StateMixin, ModuleHelper):
+    pass

plugins/module_utils/module_helper.py

@@ -6,506 +6,13 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-from functools import partial, wraps
-import traceback
 
-from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.community.general.plugins.module_utils.mh.module_helper import (
-from ansible.module_utils.common.dict_transformations import dict_merge
+    ModuleHelper, StateModuleHelper, CmdModuleHelper, CmdStateModuleHelper, AnsibleModule
-
+)
-
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.cmd import CmdMixin, ArgFormat
-class ModuleHelperException(Exception):
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.state import StateMixin
-    @staticmethod
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.deps import DependencyCtxMgr
-    def _get_remove(key, kwargs):
+from ansible_collections.community.general.plugins.module_utils.mh.exceptions import ModuleHelperException
-        if key in kwargs:
+from ansible_collections.community.general.plugins.module_utils.mh.deco import cause_changes, module_fails_on_exception
-            result = kwargs[key]
+from ansible_collections.community.general.plugins.module_utils.mh.mixins.vars import VarMeta, VarDict
-            del kwargs[key]
-            return result
-        return None
-
-    def __init__(self, *args, **kwargs):
-        self.msg = self._get_remove('msg', kwargs) or "Module failed with exception: {0}".format(self)
-        self.update_output = self._get_remove('update_output', kwargs) or {}
-        super(ModuleHelperException, self).__init__(*args)
-
-
-class ArgFormat(object):
-    """
-    Argument formatter for use as a command line parameter. Used in CmdMixin.
-    """
-    BOOLEAN = 0
-    PRINTF = 1
-    FORMAT = 2
-
-    @staticmethod
-    def stars_deco(num):
-        if num == 1:
-            def deco(f):
-                return lambda v: f(*v)
-            return deco
-        elif num == 2:
-            def deco(f):
-                return lambda v: f(**v)
-            return deco
-
-        return lambda f: f
-
-    def __init__(self, name, fmt=None, style=FORMAT, stars=0):
-        """
-        Creates a CLI-formatter for one specific argument. The argument may be a module parameter or just a named parameter for
-        the CLI command execution.
-        :param name: Name of the argument to be formatted
-        :param fmt: Either a str to be formatted (using or not printf-style) or a callable that does that
-        :param style: Whether arg_format (as str) should use printf-style formatting.
-                             Ignored if arg_format is None or not a str (should be callable).
-        :param stars: A int with 0, 1 or 2 value, indicating to formatting the value as: value, *value or **value
-        """
-        def printf_fmt(_fmt, v):
-            try:
-                return [_fmt % v]
-            except TypeError as e:
-                if e.args[0] != 'not all arguments converted during string formatting':
-                    raise
-                return [_fmt]
-
-        _fmts = {
-            ArgFormat.BOOLEAN: lambda _fmt, v: ([_fmt] if bool(v) else []),
-            ArgFormat.PRINTF: printf_fmt,
-            ArgFormat.FORMAT: lambda _fmt, v: [_fmt.format(v)],
-        }
-
-        self.name = name
-        self.stars = stars
-
-        if fmt is None:
-            fmt = "{0}"
-            style = ArgFormat.FORMAT
-
-        if isinstance(fmt, str):
-            func = _fmts[style]
-            self.arg_format = partial(func, fmt)
-        elif isinstance(fmt, list) or isinstance(fmt, tuple):
-            self.arg_format = lambda v: [_fmts[style](f, v)[0] for f in fmt]
-        elif hasattr(fmt, '__call__'):
-            self.arg_format = fmt
-        else:
-            raise TypeError('Parameter fmt must be either: a string, a list/tuple of '
-                            'strings or a function: type={0}, value={1}'.format(type(fmt), fmt))
-
-        if stars:
-            self.arg_format = (self.stars_deco(stars))(self.arg_format)
-
-    def to_text(self, value):
-        if value is None:
-            return []
-        func = self.arg_format
-        return [str(p) for p in func(value)]
-
-
-def cause_changes(on_success=None, on_failure=None):
-
-    def deco(func):
-        if on_success is None and on_failure is None:
-            return func
-
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            try:
-                self = args[0]
-                func(*args, **kwargs)
-                if on_success is not None:
-                    self.changed = on_success
-            except Exception:
-                if on_failure is not None:
-                    self.changed = on_failure
-                raise
-
-        return wrapper
-
-    return deco
-
-
-def module_fails_on_exception(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        try:
-            func(self, *args, **kwargs)
-        except SystemExit:
-            raise
-        except ModuleHelperException as e:
-            if e.update_output:
-                self.update_output(e.update_output)
-            self.module.fail_json(msg=e.msg, exception=traceback.format_exc(),
-                                  output=self.output, vars=self.vars.output(), **self.output)
-        except Exception as e:
-            msg = "Module failed with exception: {0}".format(str(e).strip())
-            self.module.fail_json(msg=msg, exception=traceback.format_exc(),
-                                  output=self.output, vars=self.vars.output(), **self.output)
-    return wrapper
-
-
-class DependencyCtxMgr(object):
-    def __init__(self, name, msg=None):
-        self.name = name
-        self.msg = msg
-        self.has_it = False
-        self.exc_type = None
-        self.exc_val = None
-        self.exc_tb = None
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.has_it = exc_type is None
-        self.exc_type = exc_type
-        self.exc_val = exc_val
-        self.exc_tb = exc_tb
-        return not self.has_it
-
-    @property
-    def text(self):
-        return self.msg or str(self.exc_val)
-
-
-class VarMeta(object):
-    NOTHING = object()
-
-    def __init__(self, diff=False, output=True, change=None, fact=False):
-        self.init = False
-        self.initial_value = None
-        self.value = None
-
-        self.diff = diff
-        self.change = diff if change is None else change
-        self.output = output
-        self.fact = fact
-
-    def set(self, diff=None, output=None, change=None, fact=None, initial_value=NOTHING):
-        if diff is not None:
-            self.diff = diff
-        if output is not None:
-            self.output = output
-        if change is not None:
-            self.change = change
-        if fact is not None:
-            self.fact = fact
-        if initial_value is not self.NOTHING:
-            self.initial_value = initial_value
-
-    def set_value(self, value):
-        if not self.init:
-            self.initial_value = value
-            self.init = True
-        self.value = value
-        return self
-
-    @property
-    def has_changed(self):
-        return self.change and (self.initial_value != self.value)
-
-    @property
-    def diff_result(self):
-        return None if not (self.diff and self.has_changed) else {
-            'before': self.initial_value,
-            'after': self.value,
-        }
-
-    def __str__(self):
-        return "<VarMeta: value={0}, initial={1}, diff={2}, output={3}, change={4}>".format(
-            self.value, self.initial_value, self.diff, self.output, self.change
-        )
-
-
-class ModuleHelper(object):
-    _output_conflict_list = ('msg', 'exception', 'output', 'vars', 'changed')
-    _dependencies = []
-    module = None
-    facts_name = None
-    output_params = ()
-    diff_params = ()
-    change_params = ()
-    facts_params = ()
-
-    class VarDict(object):
-        def __init__(self):
-            self._data = dict()
-            self._meta = dict()
-
-        def __getitem__(self, item):
-            return self._data[item]
-
-        def __setitem__(self, key, value):
-            self.set(key, value)
-
-        def __getattr__(self, item):
-            try:
-                return self._data[item]
-            except KeyError:
-                return getattr(self._data, item)
-
-        def __setattr__(self, key, value):
-            if key in ('_data', '_meta'):
-                super(ModuleHelper.VarDict, self).__setattr__(key, value)
-            else:
-                self.set(key, value)
-
-        def meta(self, name):
-            return self._meta[name]
-
-        def set_meta(self, name, **kwargs):
-            self.meta(name).set(**kwargs)
-
-        def set(self, name, value, **kwargs):
-            if name in ('_data', '_meta'):
-                raise ValueError("Names _data and _meta are reserved for use by ModuleHelper")
-            self._data[name] = value
-            if name in self._meta:
-                meta = self.meta(name)
-            else:
-                meta = VarMeta(**kwargs)
-            meta.set_value(value)
-            self._meta[name] = meta
-
-        def output(self):
-            return dict((k, v) for k, v in self._data.items() if self.meta(k).output)
-
-        def diff(self):
-            diff_results = [(k, self.meta(k).diff_result) for k in self._data]
-            diff_results = [dr for dr in diff_results if dr[1] is not None]
-            if diff_results:
-                before = dict((dr[0], dr[1]['before']) for dr in diff_results)
-                after = dict((dr[0], dr[1]['after']) for dr in diff_results)
-                return {'before': before, 'after': after}
-            return None
-
-        def facts(self):
-            facts_result = dict((k, v) for k, v in self._data.items() if self._meta[k].fact)
-            return facts_result if facts_result else None
-
-        def change_vars(self):
-            return [v for v in self._data if self.meta(v).change]
-
-        def has_changed(self, v):
-            return self._meta[v].has_changed
-
-    def __init__(self, module=None):
-        self.vars = ModuleHelper.VarDict()
-        self._changed = False
-
-        if module:
-            self.module = module
-
-        if not isinstance(self.module, AnsibleModule):
-            self.module = AnsibleModule(**self.module)
-
-        for name, value in self.module.params.items():
-            self.vars.set(
-                name, value,
-                diff=name in self.diff_params,
-                output=name in self.output_params,
-                change=None if not self.change_params else name in self.change_params,
-                fact=name in self.facts_params,
-            )
-
-    def update_vars(self, meta=None, **kwargs):
-        if meta is None:
-            meta = {}
-        for k, v in kwargs.items():
-            self.vars.set(k, v, **meta)
-
-    def update_output(self, **kwargs):
-        self.update_vars(meta={"output": True}, **kwargs)
-
-    def update_facts(self, **kwargs):
-        self.update_vars(meta={"fact": True}, **kwargs)
-
-    def __init_module__(self):
-        pass
-
-    def __run__(self):
-        raise NotImplementedError()
-
-    def __quit_module__(self):
-        pass
-
-    def _vars_changed(self):
-        return any(self.vars.has_changed(v) for v in self.vars.change_vars())
-
-    @property
-    def changed(self):
-        return self._changed
-
-    @changed.setter
-    def changed(self, value):
-        self._changed = value
-
-    def has_changed(self):
-        return self.changed or self._vars_changed()
-
-    @property
-    def output(self):
-        result = dict(self.vars.output())
-        if self.facts_name:
-            facts = self.vars.facts()
-            if facts is not None:
-                result['ansible_facts'] = {self.facts_name: facts}
-        if self.module._diff:
-            diff = result.get('diff', {})
-            vars_diff = self.vars.diff() or {}
-            result['diff'] = dict_merge(dict(diff), vars_diff)
-
-        for varname in result:
-            if varname in self._output_conflict_list:
-                result["_" + varname] = result[varname]
-                del result[varname]
-        return result
-
-    @module_fails_on_exception
-    def run(self):
-        self.fail_on_missing_deps()
-        self.__init_module__()
-        self.__run__()
-        self.__quit_module__()
-        self.module.exit_json(changed=self.has_changed(), **self.output)
-
-    @classmethod
-    def dependency(cls, name, msg):
-        cls._dependencies.append(DependencyCtxMgr(name, msg))
-        return cls._dependencies[-1]
-
-    def fail_on_missing_deps(self):
-        for d in self._dependencies:
-            if not d.has_it:
-                self.module.fail_json(changed=False,
-                                      exception="\n".join(traceback.format_exception(d.exc_type, d.exc_val, d.exc_tb)),
-                                      msg=d.text,
-                                      **self.output)
-
-
-class StateMixin(object):
-    state_param = 'state'
-    default_state = None
-
-    def _state(self):
-        state = self.module.params.get(self.state_param)
-        return self.default_state if state is None else state
-
-    def _method(self, state):
-        return "{0}_{1}".format(self.state_param, state)
-
-    def __run__(self):
-        state = self._state()
-        self.vars.state = state
-
-        # resolve aliases
-        if state not in self.module.params:
-            aliased = [name for name, param in self.module.argument_spec.items() if state in param.get('aliases', [])]
-            if aliased:
-                state = aliased[0]
-                self.vars.effective_state = state
-
-        method = self._method(state)
-        if not hasattr(self, method):
-            return self.__state_fallback__()
-        func = getattr(self, method)
-        return func()
-
-    def __state_fallback__(self):
-        raise ValueError("Cannot find method: {0}".format(self._method(self._state())))
-
-
-class CmdMixin(object):
-    """
-    Mixin for mapping module options to running a CLI command with its arguments.
-    """
-    command = None
-    command_args_formats = {}
-    run_command_fixed_options = {}
-    check_rc = False
-    force_lang = "C"
-
-    @property
-    def module_formats(self):
-        result = {}
-        for param in self.module.params.keys():
-            result[param] = ArgFormat(param)
-        return result
-
-    @property
-    def custom_formats(self):
-        result = {}
-        for param, fmt_spec in self.command_args_formats.items():
-            result[param] = ArgFormat(param, **fmt_spec)
-        return result
-
-    def _calculate_args(self, extra_params=None, params=None):
-        def add_arg_formatted_param(_cmd_args, arg_format, _value):
-            args = list(arg_format.to_text(_value))
-            return _cmd_args + args
-
-        def find_format(_param):
-            return self.custom_formats.get(_param, self.module_formats.get(_param))
-
-        extra_params = extra_params or dict()
-        cmd_args = list([self.command]) if isinstance(self.command, str) else list(self.command)
-        try:
-            cmd_args[0] = self.module.get_bin_path(cmd_args[0], required=True)
-        except ValueError:
-            pass
-        param_list = params if params else self.module.params.keys()
-
-        for param in param_list:
-            if isinstance(param, dict):
-                if len(param) != 1:
-                    raise ModuleHelperException("run_command parameter as a dict must "
-                                                "contain only one key: {0}".format(param))
-                _param = list(param.keys())[0]
-                fmt = find_format(_param)
-                value = param[_param]
-            elif isinstance(param, str):
-                if param in self.module.argument_spec:
-                    fmt = find_format(param)
-                    value = self.module.params[param]
-                elif param in extra_params:
-                    fmt = find_format(param)
-                    value = extra_params[param]
-                else:
-                    self.module.deprecate("Cannot determine value for parameter: {0}. "
-                                          "From version 4.0.0 onwards this will generate an exception".format(param),
-                                          version="4.0.0", collection_name="community.general")
-                    continue
-
-            else:
-                raise ModuleHelperException("run_command parameter must be either a str or a dict: {0}".format(param))
-            cmd_args = add_arg_formatted_param(cmd_args, fmt, value)
-
-        return cmd_args
-
-    def process_command_output(self, rc, out, err):
-        return rc, out, err
-
-    def run_command(self, extra_params=None, params=None, *args, **kwargs):
-        self.vars.cmd_args = self._calculate_args(extra_params, params)
-        options = dict(self.run_command_fixed_options)
-        env_update = dict(options.get('environ_update', {}))
-        options['check_rc'] = options.get('check_rc', self.check_rc)
-        if self.force_lang:
-            env_update.update({'LANGUAGE': self.force_lang})
-            self.update_output(force_lang=self.force_lang)
-            options['environ_update'] = env_update
-        options.update(kwargs)
-        rc, out, err = self.module.run_command(self.vars.cmd_args, *args, **options)
-        self.update_output(rc=rc, stdout=out, stderr=err)
-        return self.process_command_output(rc, out, err)
-
-
-class StateModuleHelper(StateMixin, ModuleHelper):
-    pass
-
-
-class CmdModuleHelper(CmdMixin, ModuleHelper):
-    pass
-
-
-class CmdStateModuleHelper(CmdMixin, StateMixin, ModuleHelper):
-    pass

tests/unit/plugins/module_utils/test_module_helper.py

@@ -6,12 +6,10 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import namedtuple
-
 import pytest
 
 from ansible_collections.community.general.plugins.module_utils.module_helper import (
-    ArgFormat, DependencyCtxMgr, ModuleHelper, VarMeta, cause_changes
+    ArgFormat, DependencyCtxMgr, VarMeta, VarDict, cause_changes
 )
 
 
@@ -144,7 +142,7 @@ def test_variable_meta_diff():
 
 
 def test_vardict():
-    vd = ModuleHelper.VarDict()
+    vd = VarDict()
     vd.set('a', 123)
     assert vd['a'] == 123
     assert vd.a == 123

tests/unit/plugins/modules/system/test_xfconf.py

@@ -21,7 +21,7 @@ def patch_xfconf(mocker):
     """
     Function used for mocking some parts of redhat_subscribtion module
     """
-    mocker.patch('ansible_collections.community.general.plugins.module_utils.module_helper.AnsibleModule.get_bin_path',
+    mocker.patch('ansible_collections.community.general.plugins.module_utils.mh.module_helper.AnsibleModule.get_bin_path',
                  return_value='/testbin/xfconf-query')
 
 
@@ -332,7 +332,7 @@ def test_xfconf(mocker, capfd, patch_xfconf, testcase):
     # Mock function used for running commands first
     call_results = [item[2] for item in testcase['run_command.calls']]
     mock_run_command = mocker.patch(
-        'ansible_collections.community.general.plugins.module_utils.module_helper.AnsibleModule.run_command',
+        'ansible_collections.community.general.plugins.module_utils.mh.module_helper.AnsibleModule.run_command',
         side_effect=call_results)
 
     # Try to run test case