docker_swarm_service: Enable tests (#51170)

* Enable tests

* Comment fixes

* Try lowering timeouts

* Comment fix

* Comment fix

* Comment fix

* Add a pause to let service update

* Fix comment

* Disable dns_search tests

* Disable dns_servers test

* Fix comment

* Fix comment

* Fix comment

* Fix comment

* Fix comment

* Fix comment

* Fix comment

* Revert "Disable dns_servers test"

This reverts commit 763e9da716b78f4986f313b3ba1ab98faacb742e.

* Revert "Disable dns_search tests"

This reverts commit 2859e4e3a5ebdca078de84d821bb53bbdf967dfd.

* Revert "Add a pause to let service update"

This reverts commit e990dfae1a62e9a42b07960819818bc75fd04427.

* Revert "Try lowering timeouts"

This reverts commit 1617772de81ecef0e560b38c7564646ec3874c3c.

* Ensure that services are running while testing

* Retry tasks on update out of sequence error

* Remove unnecessary check for APIError.explanation

Co-Authored-By: hannseman <hannes@5monkeys.se>

* Ignore errors when tearing down test suite

* Retry with a loop instead of tail recursion

* Initialize self.diff_trace in run

* Add change log fragment

* Actually raise error

* Add unit test for retrying

* Lint

* Change to bugfix

* Remove whitespace

* Mock docker dependency

* Use download.fedoraproject.org

* Revert "Use download.fedoraproject.org"

This reverts commit 5931791f7cfdd339f15068c8706b39905517abdf.

changelogs/fragments/51170-docker_swarm_service-handle-out-of-sequence-error.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - "docker_swarm_service - When docker fails to update a container with an ``update out of sequence`` error, the module will retry to update up to two times, and only fail if all three attempts do not succeed."

lib/ansible/modules/cloud/docker/docker_swarm_service.py

@@ -542,7 +542,7 @@ try:
     from distutils.version import LooseVersion
     from docker import types
     from docker.utils import parse_repository_tag
-    from docker.errors import DockerException
+    from docker.errors import APIError, DockerException
 except Exception:
     # missing docker-py handled in ansible.module_utils.docker
     pass
@@ -1209,9 +1209,11 @@ class DockerServiceManager():
 
     def __init__(self, client):
         self.client = client
-        self.diff_tracker = DifferenceTracker()
+        self.retries = 2
+        self.diff_tracker = None
 
     def run(self):
+        self.diff_tracker = DifferenceTracker()
         module = self.client.module
 
         image = module.params['image']
@@ -1294,6 +1296,19 @@ class DockerServiceManager():
 
         return msg, changed, rebuilt, differences.get_legacy_docker_diffs(), facts
 
+    def run_safe(self):
+        while True:
+            try:
+                return self.run()
+            except APIError as e:
+                # Sometimes Version.Index will have changed between an inspect and
+                # update. If this is encountered we'll retry the update.
+                if self.retries > 0 and 'update out of sequence' in str(e.explanation):
+                    self.retries -= 1
+                    time.sleep(1)
+                else:
+                    raise
+
 
 def _detect_publish_mode_usage(client):
     for publish_def in client.module.params['publish']:
@@ -1385,7 +1400,7 @@ def main():
     )
 
     dsm = DockerServiceManager(client)
-    msg, changed, rebuilt, changes, facts = dsm.run()
+    msg, changed, rebuilt, changes, facts = dsm.run_safe()
 
     results = dict(
         msg=msg,

test/integration/targets/docker_swarm_service/aliases

@@ -3,4 +3,3 @@ skip/osx
 skip/freebsd
 destructive
 skip/rhel8.0
-unstable

test/units/modules/cloud/docker/test_docker_swarm_service.py

@@ -0,0 +1,59 @@
+import pytest
+
+
+class APIErrorMock(Exception):
+
+    def __init__(self, message, response=None, explanation=None):
+        self.message = message
+        self.response = response
+        self.explanation = explanation
+
+
+@pytest.fixture(autouse=True)
+def docker_module_mock(mocker):
+    docker_module_mock = mocker.MagicMock()
+    docker_utils_module_mock = mocker.MagicMock()
+    docker_errors_module_mock = mocker.MagicMock()
+    docker_errors_module_mock.APIError = APIErrorMock
+    mock_modules = {
+        'docker': docker_module_mock,
+        'docker.utils': docker_utils_module_mock,
+        'docker.errors': docker_errors_module_mock,
+    }
+    return mocker.patch.dict('sys.modules', **mock_modules)
+
+
+@pytest.fixture(autouse=True)
+def docker_swarm_service():
+    from ansible.modules.cloud.docker import docker_swarm_service
+    return docker_swarm_service
+
+
+def test_retry_on_out_of_sequence_error(mocker, docker_swarm_service):
+    run_mock = mocker.MagicMock(
+        side_effect=APIErrorMock(
+            message='',
+            response=None,
+            explanation='rpc error: code = Unknown desc = update out of sequence',
+        )
+    )
+    manager = docker_swarm_service.DockerServiceManager(client=None)
+    manager.run = run_mock
+    with pytest.raises(APIErrorMock):
+        manager.run_safe()
+    assert run_mock.call_count == 3
+
+
+def test_no_retry_on_general_api_error(mocker, docker_swarm_service):
+    run_mock = mocker.MagicMock(
+        side_effect=APIErrorMock(
+            message='',
+            response=None,
+            explanation='some error',
+        )
+    )
+    manager = docker_swarm_service.DockerServiceManager(client=None)
+    manager.run = run_mock
+    with pytest.raises(APIErrorMock):
+        manager.run_safe()
+    assert run_mock.call_count == 1