From b67719ba1d28dbb3524893054555319df99625ed Mon Sep 17 00:00:00 2001 From: Dave Bendit Date: Wed, 12 Dec 2018 03:05:12 -0600 Subject: [PATCH] Docker common consolidation (#49707) * [docker] Consolidating Python Boolean conversion for Docker API (#49563) * [docker] Consolidating docker option min version checks (#49564) * [docker] Moving option min version checks out of docker_swarm (#49564) Also renaming Boolean cleanup function and fixing docker_container minimum version check for network interfaces. * Cleanup from PR feedback --- lib/ansible/module_utils/docker_common.py | 78 ++++++++++++- .../modules/cloud/docker/docker_container.py | 110 ++++++------------ .../modules/cloud/docker/docker_network.py | 82 ++----------- .../modules/cloud/docker/docker_prune.py | 26 +---- .../modules/cloud/docker/docker_swarm.py | 63 ++-------- 5 files changed, 132 insertions(+), 227 deletions(-) diff --git a/lib/ansible/module_utils/docker_common.py b/lib/ansible/module_utils/docker_common.py index d54dd9621d..a0d5a77a74 100644 --- a/lib/ansible/module_utils/docker_common.py +++ b/lib/ansible/module_utils/docker_common.py @@ -163,7 +163,8 @@ class AnsibleDockerClient(Client): def __init__(self, argument_spec=None, supports_check_mode=False, mutually_exclusive=None, required_together=None, required_if=None, min_docker_version=MIN_DOCKER_VERSION, - min_docker_api_version=None): + min_docker_api_version=None, option_minimal_versions=None, + option_minimal_versions_ignore_params=None): merged_arg_spec = dict() merged_arg_spec.update(DOCKER_COMMON_ARGS) @@ -235,6 +236,9 @@ class AnsibleDockerClient(Client): if self.docker_api_version < LooseVersion(min_docker_api_version): self.fail('docker API version is %s. Minimum version required is %s.' % (self.docker_api_version_str, min_docker_api_version)) + if option_minimal_versions is not None: + self._get_minimal_versions(option_minimal_versions, option_minimal_versions_ignore_params) + def log(self, msg, pretty_print=False): pass # if self.debug: @@ -416,6 +420,58 @@ class AnsibleDockerClient(Client): % (self.auth_params['tls_hostname'], match.group(1), match.group(1))) self.fail("SSL Exception: %s" % (error)) + def _get_minimal_versions(self, option_minimal_versions, ignore_params=None): + self.option_minimal_versions = dict() + for option in self.module.argument_spec: + if ignore_params is not None: + if option in ignore_params: + continue + self.option_minimal_versions[option] = dict() + self.option_minimal_versions.update(option_minimal_versions) + + for option, data in self.option_minimal_versions.items(): + # Test whether option is supported, and store result + support_docker_py = True + support_docker_api = True + if 'docker_py_version' in data: + support_docker_py = self.docker_py_version >= LooseVersion(data['docker_py_version']) + if 'docker_api_version' in data: + support_docker_api = self.docker_api_version >= LooseVersion(data['docker_api_version']) + data['supported'] = support_docker_py and support_docker_api + # Fail if option is not supported but used + if not data['supported']: + # Test whether option is specified + if 'detect_usage' in data: + used = data['detect_usage']() + else: + used = self.module.params.get(option) is not None + if used and 'default' in self.module.argument_spec[option]: + used = self.module.params[option] != self.module.argument_spec[option]['default'] + if used: + # If the option is used, compose error message. + if 'usage_msg' in data: + usg = data['usage_msg'] + else: + usg = 'set %s option' % (option, ) + if not support_docker_api: + msg = 'docker API version is %s. Minimum version required is %s to %s.' + msg = msg % (self.docker_api_version_str, data['docker_api_version'], usg) + elif not support_docker_py: + if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'): + msg = ("docker-py version is %s. Minimum version required is %s to %s. " + "Consider switching to the 'docker' package if you do not require Python 2.6 support.") + elif self.docker_py_version < LooseVersion('2.0.0'): + msg = ("docker-py version is %s. Minimum version required is %s to %s. " + "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before " + "installing 'docker' to avoid a broken installation.") + else: + msg = "docker version is %s. Minimum version required is %s to %s." + msg = msg % (docker_version, data['docker_py_version'], usg) + else: + # should not happen + msg = 'Cannot %s with your configuration.' % (usg, ) + self.fail(msg) + def get_container(self, name=None): ''' Lookup a container and return the inspection results. @@ -741,3 +797,23 @@ class DifferenceTracker(object): ''' result = [entry['name'] for entry in self._diff] return result + + +def clean_dict_booleans_for_docker_api(data): + ''' + Go doesn't like Python booleans 'True' or 'False', while Ansible is just + fine with them in YAML. As such, they need to be converted in cases where + we pass dictionaries to the Docker API (e.g. docker_network's + driver_options and docker_prune's filters). + ''' + result = dict() + if data is not None: + for k, v in data.items(): + if v is True: + v = 'true' + elif v is False: + v = 'false' + else: + v = str(v) + result[str(k)] = v + return result diff --git a/lib/ansible/modules/cloud/docker/docker_container.py b/lib/ansible/modules/cloud/docker/docker_container.py index bf0080b7f0..81687dbb3b 100644 --- a/lib/ansible/modules/cloud/docker/docker_container.py +++ b/lib/ansible/modules/cloud/docker/docker_container.py @@ -2745,27 +2745,43 @@ class AnsibleDockerClientContainer(AnsibleDockerClient): self.module.warn('The ignore_image option has been overridden by the comparisons option!') self.comparisons = comparisons - def _get_minimal_versions(self): - # Helper function to detect whether any specified network uses ipv4_address or ipv6_address + def _get_additional_minimal_versions(self): + stop_timeout_supported = self.docker_api_version >= LooseVersion('1.25') + stop_timeout_needed_for_update = self.module.params.get("stop_timeout") is not None and self.module.params.get('state') != 'absent' + if stop_timeout_supported: + stop_timeout_supported = self.docker_py_version >= LooseVersion('2.1') + if stop_timeout_needed_for_update and not stop_timeout_supported: + # We warn (instead of fail) since in older versions, stop_timeout was not used + # to update the container's configuration, but only when stopping a container. + self.module.warn("docker or docker-py version is %s. Minimum version required is 2.1 to update " + "the container's stop_timeout configuration. " + "If you use the 'docker-py' module, you have to switch to the docker 'Python' package." % (docker_version,)) + else: + if stop_timeout_needed_for_update and not stop_timeout_supported: + # We warn (instead of fail) since in older versions, stop_timeout was not used + # to update the container's configuration, but only when stopping a container. + self.module.warn("docker API version is %s. Minimum version required is 1.25 to set or " + "update the container's stop_timeout configuration." % (self.docker_api_version_str,)) + self.option_minimal_versions['stop_timeout']['supported'] = stop_timeout_supported + + def __init__(self, **kwargs): def detect_ipvX_address_usage(): + ''' + Helper function to detect whether any specified network uses ipv4_address or ipv6_address + ''' for network in self.module.params.get("networks") or []: if network.get('ipv4_address') is not None or network.get('ipv6_address') is not None: return True return False - self.option_minimal_versions = dict( + option_minimal_versions = dict( # internal options log_config=dict(), publish_all_ports=dict(), ports=dict(), volume_binds=dict(), name=dict(), - ) - for option, data in self.module.argument_spec.items(): - if option in self.__NON_CONTAINER_PROPERTY_OPTIONS: - continue - self.option_minimal_versions[option] = dict() - self.option_minimal_versions.update(dict( + # normal options device_read_bps=dict(docker_py_version='1.9.0', docker_api_version='1.22'), device_read_iops=dict(docker_py_version='1.9.0', docker_api_version='1.22'), device_write_bps=dict(docker_py_version='1.9.0', docker_api_version='1.22'), @@ -2791,74 +2807,16 @@ class AnsibleDockerClientContainer(AnsibleDockerClient): pids_limit=dict(docker_py_version='1.10.0', docker_api_version='1.23'), # specials ipvX_address_supported=dict(docker_py_version='1.9.0', detect_usage=detect_ipvX_address_usage, - usage_msg='ipv4_address or ipv6_address in networks'), - stop_timeout=dict(), # see below! - )) + usage_msg='ipv4_address or ipv6_address in networks'), # see above + stop_timeout=dict(), # see _get_additional_minimal_versions() + ) - for option, data in self.option_minimal_versions.items(): - # Test whether option is supported, and store result - support_docker_py = True - support_docker_api = True - if 'docker_py_version' in data: - support_docker_py = self.docker_py_version >= LooseVersion(data['docker_py_version']) - if 'docker_api_version' in data: - support_docker_api = self.docker_api_version >= LooseVersion(data['docker_api_version']) - data['supported'] = support_docker_py and support_docker_api - # Fail if option is not supported but used - if not data['supported']: - # Test whether option is specified - if 'detect_usage' in data: - used = data['detect_usage']() - else: - used = self.module.params.get(option) is not None - if used and 'default' in self.module.argument_spec[option]: - used = self.module.params[option] != self.module.argument_spec[option]['default'] - if used: - # If the option is used, compose error message. - if 'usage_msg' in data: - usg = data['usage_msg'] - else: - usg = 'set %s option' % (option, ) - if not support_docker_api: - msg = 'docker API version is %s. Minimum version required is %s to %s.' - msg = msg % (self.docker_api_version_str, data['docker_api_version'], usg) - elif not support_docker_py: - if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'): - msg = ("docker-py version is %s. Minimum version required is %s to %s. " - "Consider switching to the 'docker' package if you do not require Python 2.6 support.") - elif self.docker_py_version < LooseVersion('2.0.0'): - msg = ("docker-py version is %s. Minimum version required is %s to %s. " - "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before " - "installing 'docker' to avoid a broken installation.") - else: - msg = "docker version is %s. Minimum version required is %s to %s." - msg = msg % (docker_version, data['docker_py_version'], usg) - else: - # should not happen - msg = 'Cannot %s with your configuration.' % (usg, ) - self.fail(msg) - - stop_timeout_supported = self.docker_api_version >= LooseVersion('1.25') - stop_timeout_needed_for_update = self.module.params.get("stop_timeout") is not None and self.module.params.get('state') != 'absent' - if stop_timeout_supported: - stop_timeout_supported = self.docker_py_version >= LooseVersion('2.1') - if stop_timeout_needed_for_update and not stop_timeout_supported: - # We warn (instead of fail) since in older versions, stop_timeout was not used - # to update the container's configuration, but only when stopping a container. - self.module.warn("docker or docker-py version is %s. Minimum version required is 2.1 to update " - "the container's stop_timeout configuration. " - "If you use the 'docker-py' module, you have to switch to the docker 'Python' package." % (docker_version,)) - else: - if stop_timeout_needed_for_update and not stop_timeout_supported: - # We warn (instead of fail) since in older versions, stop_timeout was not used - # to update the container's configuration, but only when stopping a container. - self.module.warn("docker API version is %s. Minimum version required is 1.25 to set or " - "update the container's stop_timeout configuration." % (self.docker_api_version_str,)) - self.option_minimal_versions['stop_timeout']['supported'] = stop_timeout_supported - - def __init__(self, **kwargs): - super(AnsibleDockerClientContainer, self).__init__(**kwargs) - self._get_minimal_versions() + super(AnsibleDockerClientContainer, self).__init__( + option_minimal_versions=option_minimal_versions, + option_minimal_versions_ignore_params=self.__NON_CONTAINER_PROPERTY_OPTIONS, + **kwargs + ) + self._get_additional_minimal_versions() self._parse_comparisons() diff --git a/lib/ansible/modules/cloud/docker/docker_network.py b/lib/ansible/modules/cloud/docker/docker_network.py index 2c1bff5ba9..d9ddd3316e 100644 --- a/lib/ansible/modules/cloud/docker/docker_network.py +++ b/lib/ansible/modules/cloud/docker/docker_network.py @@ -255,6 +255,7 @@ from ansible.module_utils.docker_common import ( DockerBaseClass, docker_version, DifferenceTracker, + clean_dict_booleans_for_docker_api, ) try: @@ -315,77 +316,8 @@ def get_ip_version(cidr): raise ValueError('"{0}" is not a valid CIDR'.format(cidr)) -def get_driver_options(driver_options): - # TODO: Move this and the same from docker_prune.py to docker_common.py - result = dict() - if driver_options is not None: - for k, v in driver_options.items(): - # Go doesn't like 'True' or 'False' - if v is True: - v = 'true' - elif v is False: - v = 'false' - else: - v = str(v) - result[str(k)] = v - return result - - class DockerNetworkManager(object): - def _get_minimal_versions(self): - # TODO: Move this and the same from docker_container.py to docker_common.py - self.option_minimal_versions = dict() - for option, data in self.client.module.argument_spec.items(): - self.option_minimal_versions[option] = dict() - self.option_minimal_versions.update(dict( - scope=dict(docker_py_version='2.6.0', docker_api_version='1.30'), - attachable=dict(docker_py_version='2.0.0', docker_api_version='1.26'), - )) - - for option, data in self.option_minimal_versions.items(): - # Test whether option is supported, and store result - support_docker_py = True - support_docker_api = True - if 'docker_py_version' in data: - support_docker_py = self.client.docker_py_version >= LooseVersion(data['docker_py_version']) - if 'docker_api_version' in data: - support_docker_api = self.client.docker_api_version >= LooseVersion(data['docker_api_version']) - data['supported'] = support_docker_py and support_docker_api - # Fail if option is not supported but used - if not data['supported']: - # Test whether option is specified - if 'detect_usage' in data: - used = data['detect_usage']() - else: - used = self.client.module.params.get(option) is not None - if used and 'default' in self.client.module.argument_spec[option]: - used = self.client.module.params[option] != self.client.module.argument_spec[option]['default'] - if used: - # If the option is used, compose error message. - if 'usage_msg' in data: - usg = data['usage_msg'] - else: - usg = 'set %s option' % (option, ) - if not support_docker_api: - msg = 'docker API version is %s. Minimum version required is %s to %s.' - msg = msg % (self.client.docker_api_version_str, data['docker_api_version'], usg) - elif not support_docker_py: - if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'): - msg = ("docker-py version is %s. Minimum version required is %s to %s. " - "Consider switching to the 'docker' package if you do not require Python 2.6 support.") - elif self.client.docker_py_version < LooseVersion('2.0.0'): - msg = ("docker-py version is %s. Minimum version required is %s to %s. " - "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before " - "installing 'docker' to avoid a broken installation.") - else: - msg = "docker version is %s. Minimum version required is %s to %s." - msg = msg % (docker_version, data['docker_py_version'], usg) - else: - # should not happen - msg = 'Cannot %s with your configuration.' % (usg, ) - self.client.fail(msg) - def __init__(self, client): self.client = client self.parameters = TaskParameters(client) @@ -398,8 +330,6 @@ class DockerNetworkManager(object): self.diff_tracker = DifferenceTracker() self.diff_result = dict() - self._get_minimal_versions() - self.existing_network = self.get_existing_network() if not self.parameters.connected and self.existing_network: @@ -410,7 +340,7 @@ class DockerNetworkManager(object): self.parameters.ipam_config = [self.parameters.ipam_options] if self.parameters.driver_options: - self.parameters.driver_options = get_driver_options(self.parameters.driver_options) + self.parameters.driver_options = clean_dict_booleans_for_docker_api(self.parameters.driver_options) state = self.parameters.state if state == 'present': @@ -665,13 +595,19 @@ def main(): ('ipam_config', 'ipam_options') ] + option_minimal_versions = dict( + scope=dict(docker_py_version='2.6.0', docker_api_version='1.30'), + attachable=dict(docker_py_version='2.0.0', docker_api_version='1.26'), + ) + client = AnsibleDockerClient( argument_spec=argument_spec, mutually_exclusive=mutually_exclusive, supports_check_mode=True, min_docker_version='1.10.0', - min_docker_api_version='1.22' + min_docker_api_version='1.22', # "The docker server >= 1.10.0" + option_minimal_versions=option_minimal_versions, ) cm = DockerNetworkManager(client) diff --git a/lib/ansible/modules/cloud/docker/docker_prune.py b/lib/ansible/modules/cloud/docker/docker_prune.py index 5f2e82d142..2ff736c0b8 100644 --- a/lib/ansible/modules/cloud/docker/docker_prune.py +++ b/lib/ansible/modules/cloud/docker/docker_prune.py @@ -176,28 +176,12 @@ from distutils.version import LooseVersion from ansible.module_utils.docker_common import AnsibleDockerClient try: - from ansible.module_utils.docker_common import docker_version + from ansible.module_utils.docker_common import docker_version, clean_dict_booleans_for_docker_api except Exception as dummy: # missing docker-py handled in ansible.module_utils.docker pass -def get_filters(module, name): - result = dict() - filters = module.params.get(name) - if filters is not None: - for k, v in filters.items(): - # Go doesn't like 'True' or 'False' - if v is True: - v = 'true' - elif v is False: - v = 'false' - else: - v = str(v) - result[str(k)] = v - return result - - def main(): argument_spec = dict( containers=dict(type='bool', default=False), @@ -227,24 +211,24 @@ def main(): result = dict() if client.module.params['containers']: - filters = get_filters(client.module, 'containers_filters') + filters = clean_dict_booleans_for_docker_api(client.module.params.get('containers_filters')) res = client.prune_containers(filters=filters) result['containers'] = res.get('ContainersDeleted') or [] result['containers_space_reclaimed'] = res['SpaceReclaimed'] if client.module.params['images']: - filters = get_filters(client.module, 'images_filters') + filters = clean_dict_booleans_for_docker_api(client.module.params.get('images_filters')) res = client.prune_images(filters=filters) result['images'] = res.get('ImagesDeleted') or [] result['images_space_reclaimed'] = res['SpaceReclaimed'] if client.module.params['networks']: - filters = get_filters(client.module, 'networks_filters') + filters = clean_dict_booleans_for_docker_api(client.module.params.get('networks_filters')) res = client.prune_networks(filters=filters) result['networks'] = res.get('NetworksDeleted') or [] if client.module.params['volumes']: - filters = get_filters(client.module, 'volumes_filters') + filters = clean_dict_booleans_for_docker_api(client.module.params.get('volumes_filters')) res = client.prune_volumes(filters=filters) result['volumes'] = res.get('VolumesDeleted') or [] result['volumes_space_reclaimed'] = res['SpaceReclaimed'] diff --git a/lib/ansible/modules/cloud/docker/docker_swarm.py b/lib/ansible/modules/cloud/docker/docker_swarm.py index baf4bcface..ebacc4c0b8 100644 --- a/lib/ansible/modules/cloud/docker/docker_swarm.py +++ b/lib/ansible/modules/cloud/docker/docker_swarm.py @@ -288,60 +288,6 @@ class TaskParameters(DockerBaseClass): class SwarmManager(DockerBaseClass): - def _get_minimal_versions(self): - # TODO: Move this and the same from docker_container.py to docker_common.py - self.option_minimal_versions = dict() - for option, data in self.client.module.argument_spec.items(): - self.option_minimal_versions[option] = dict() - self.option_minimal_versions.update(dict( - signing_ca_cert=dict(docker_api_version='1.30'), - signing_ca_key=dict(docker_api_version='1.30'), - ca_force_rotate=dict(docker_api_version='1.30'), - )) - - for option, data in self.option_minimal_versions.items(): - # Test whether option is supported, and store result - support_docker_py = True - support_docker_api = True - if 'docker_py_version' in data: - support_docker_py = self.client.docker_py_version >= LooseVersion(data['docker_py_version']) - if 'docker_api_version' in data: - support_docker_api = self.client.docker_api_version >= LooseVersion(data['docker_api_version']) - data['supported'] = support_docker_py and support_docker_api - # Fail if option is not supported but used - if not data['supported']: - # Test whether option is specified - if 'detect_usage' in data: - used = data['detect_usage']() - else: - used = self.client.module.params.get(option) is not None - if used and 'default' in self.client.module.argument_spec[option]: - used = self.client.module.params[option] != self.client.module.argument_spec[option]['default'] - if used: - # If the option is used, compose error message. - if 'usage_msg' in data: - usg = data['usage_msg'] - else: - usg = 'set %s option' % (option, ) - if not support_docker_api: - msg = 'docker API version is %s. Minimum version required is %s to %s.' - msg = msg % (self.client.docker_api_version_str, data['docker_api_version'], usg) - elif not support_docker_py: - if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'): - msg = ("docker-py version is %s. Minimum version required is %s to %s. " - "Consider switching to the 'docker' package if you do not require Python 2.6 support.") - elif self.client.docker_py_version < LooseVersion('2.0.0'): - msg = ("docker-py version is %s. Minimum version required is %s to %s. " - "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before " - "installing 'docker' to avoid a broken installation.") - else: - msg = "docker version is %s. Minimum version required is %s to %s." - msg = msg % (docker_version, data['docker_py_version'], usg) - else: - # should not happen - msg = 'Cannot %s with your configuration.' % (usg, ) - self.client.fail(msg) - def __init__(self, client, results): super(SwarmManager, self).__init__() @@ -350,8 +296,6 @@ class SwarmManager(DockerBaseClass): self.results = results self.check_mode = self.client.check_mode - self._get_minimal_versions() - self.parameters = TaskParameters(client) def __call__(self): @@ -562,12 +506,19 @@ def main(): ('state', 'remove', ['node_id']) ] + option_minimal_versions = dict( + signing_ca_cert=dict(docker_api_version='1.30'), + signing_ca_key=dict(docker_api_version='1.30'), + ca_force_rotate=dict(docker_api_version='1.30'), + ) + client = AnsibleDockerClient( argument_spec=argument_spec, supports_check_mode=True, required_if=required_if, min_docker_version='2.6.0', min_docker_api_version='1.25', + option_minimal_versions=option_minimal_versions, ) results = dict(