Allow subspec defaults to be processed when the parent argument is not supplied (#38967)

* Allow subspec defaults to be processed when the parent argument is not supplied

* Allow this to be configurable via apply_defaults on the parent

* Document attributes of arguments in argument_spec

* Switch manageiq_connection to use apply_defaults

* add choices to api_version in argument_spec

docs/docsite/rst/dev_guide/developing_program_flow_modules.rst

@@ -527,3 +527,111 @@ Passing arguments via stdin was chosen for the following reasons:
   systems limit the total size of the environment.  This could lead to
   truncation of the parameters if we hit that limit.
 
+
+.. _ansiblemodule:
+
+AnsibleModule
+-------------
+
+.. _argument_spec:
+
+Argument Spec
+^^^^^^^^^^^^^
+
+The ``argument_spec`` provided to ``AnsibleModule`` defines the supported arguments for a module, as well as their type, defaults and more.
+
+Example ``argument_spec``:
+
+.. code-block:: python
+
+    module = AnsibleModule(argument_spec=dict(
+        top_level=dict(
+            type='dict',
+            options=dict(
+                second_level=dict(
+                    default=True,
+                    type='bool',
+                )
+            )
+        )
+    ))
+
+This section will discss the behavioral attributes for arguments
+
+type
+~~~~
+
+``type`` allows you to define the type of the value accepted for the argument. The default value for ``type`` is ``str``. Possible values are:
+
+* str
+* list
+* dict
+* bool
+* int
+* float
+* path
+* raw
+* jsonarg
+* json
+* bytes
+* bits
+
+The ``raw`` type, performs no type validation or type casing, and maintains the type of the passed value.
+
+elements
+~~~~~~~~
+
+``elements`` works in combination with ``type`` when ``type='list'``. ``elements`` can then be defined as ``elements='int'`` or any other type, indicating that each element of the specified list should be of that type.
+
+default
+~~~~~~~
+
+The ``default`` option allows sets a default value for the argument for the scenario when the argument is not provided to the module. When not specified, the default value is ``None``.
+
+fallback
+~~~~~~~~
+
+``fallback`` accepts a ``tuple`` where the first argument is a callable (function) that will be used to perform the lookup, based on the second argument. The second argument is a list of values to be accepted by the callable.
+
+The most common callable used is ``env_fallback`` which will allow an argument to optionally use an environment variable when the argument is not supplied.
+
+Example::
+
+    username=dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME']))
+
+choices
+~~~~~~~
+
+``choices`` accepts a list of choices that the argument will accept. The types of ``choices`` should match the ``type``.
+
+required
+~~~~~~~~
+
+``required`` accepts a boolean, either ``True`` or ``False`` that indicates that the argument is required. This should not be used in combination with ``default``.
+
+no_log
+~~~~~~
+
+``no_log`` indicates that the value of the argument should not be logged or displayed.
+
+aliases
+~~~~~~~
+
+``aliases`` accepts a list of alternative argument names for the argument, such as the case where the argument is ``name`` but the module accepts ``aliases=['pkg']`` to allow ``pkg`` to be interchangably with ``name``
+
+options
+~~~~~~~
+
+``options`` implements the ability to create a sub-argument_spec, where the sub options of the top level argument are also validated using the attributes discussed in this section. The example at the top of this section demonstrates use of ``options``. ``type`` or ``elements`` should be ``dict`` is this case.
+
+apply_defaults
+~~~~~~~~~~~~~~
+
+``apply_defaults`` works alongside ``options`` and allows the ``default`` of the sub-options to be applied even when the top-level argument is not supplied.
+
+In the example of the ``argument_spec`` at the top of this section, it would allow ``module.params['top_level']['second_level']`` to be defined, even if the user does not provide ``top_level`` when calling the module.
+
+removed_in_version
+~~~~~~~~~~~~~~~~~~
+
+``removed_in_version`` indicates which version of Ansible a deprecated argument will be removed in.

lib/ansible/module_utils/basic.py

@@ -1975,7 +1975,13 @@ class AnsibleModule(object):
             wanted = v.get('type', None)
             if wanted == 'dict' or (wanted == 'list' and v.get('elements', '') == 'dict'):
                 spec = v.get('options', None)
-                if spec is None or k not in params or params[k] is None:
+                if v.get('apply_defaults', False):
+                    if spec is not None:
+                        if params.get(k) is None:
+                            params[k] = {}
+                    else:
+                        continue
+                elif spec is None or k not in params or params[k] is None:
                     continue
 
                 self._options_context.append(k)

lib/ansible/module_utils/manageiq.py

@@ -48,7 +48,7 @@ def manageiq_argument_spec():
 
     return dict(
         manageiq_connection=dict(type='dict',
-                                 default=dict(verify_ssl=True),
+                                 apply_defaults=True,
                                  options=options),
     )
 

lib/ansible/modules/remote_management/manageiq/manageiq_provider.py

@@ -801,7 +801,7 @@ def main():
         project=dict(),
         azure_tenant_id=dict(aliases=['keystone_v3_domain_id']),
         tenant_mapping_enabled=dict(default=False, type='bool'),
-        api_version=dict(),
+        api_version=dict(choices=['v2', 'v3']),
         type=dict(choices=supported_providers().keys()),
     )
     # add the manageiq connection arguments to the arguments

test/sanity/validate-modules/ignore.txt

@@ -1338,13 +1338,6 @@ lib/ansible/modules/remote_management/hpilo/hpilo_boot.py E324
 lib/ansible/modules/remote_management/hpilo/hpilo_boot.py E326
 lib/ansible/modules/remote_management/ipmi/ipmi_boot.py E326
 lib/ansible/modules/remote_management/ipmi/ipmi_power.py E326
-lib/ansible/modules/remote_management/manageiq/manageiq_alert_profiles.py E324
-lib/ansible/modules/remote_management/manageiq/manageiq_alerts.py E324
-lib/ansible/modules/remote_management/manageiq/manageiq_policies.py E324
-lib/ansible/modules/remote_management/manageiq/manageiq_provider.py E324
-lib/ansible/modules/remote_management/manageiq/manageiq_provider.py E326
-lib/ansible/modules/remote_management/manageiq/manageiq_tags.py E324
-lib/ansible/modules/remote_management/manageiq/manageiq_user.py E324
 lib/ansible/modules/remote_management/oneview/oneview_datacenter_facts.py E322
 lib/ansible/modules/remote_management/oneview/oneview_enclosure_facts.py E322
 lib/ansible/modules/remote_management/oneview/oneview_ethernet_network.py E322

test/units/module_utils/basic/test_argument_spec.py

@@ -412,6 +412,19 @@ class TestComplexOptions:
         assert isinstance(am.params['foobar']['baz'], str)
         assert am.params['foobar']['baz'] == 'test data'
 
+    @pytest.mark.parametrize('stdin,spec,expected', [
+        ({},
+         {'one': {'type': 'dict', 'apply_defaults': True, 'options': {'two': {'default': True, 'type': 'bool'}}}},
+         {'two': True}),
+        ({},
+         {'one': {'type': 'dict', 'options': {'two': {'default': True, 'type': 'bool'}}}},
+         None),
+    ], indirect=['stdin'])
+    def test_subspec_not_required_defaults(self, stdin, spec, expected):
+        # Check that top level not required, processed subspec defaults
+        am = basic.AnsibleModule(spec)
+        assert am.params['one'] == expected
+
 
 class TestLoadFileCommonArguments:
     @pytest.mark.parametrize('stdin', [{}], indirect=['stdin'])