Add fallback url for jenkins plugin (#1334)

* uncoupled updates_url from plugin download urls
added new parameters: versioned_plugins_url, latest_plugins_url

* parameters updates_url, latest_plugins_url and versioned_plugins_url changed type to list of strings to implement fallback URLs usage
added type conversion if they are string (backward compatibility)

* removed type conversion this is handled by ansible validation
fix: dont fail if first url fails

* added fallback: if installation from plugin manager fails, try downloading the plugin manually

* fixed test failures

* PEP8 indent fix

* changelog fragment

* added debug outputs for new url fallback behavior

* added version_added in description for latest_plugins_url

Co-authored-by: Felix Fontein <felix@fontein.de>

* added version_added in description for versioned_plugins_url

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update changelogs/fragments/1334-jenkins-plugin-fallback-urls.yaml

Co-authored-by: Felix Fontein <felix@fontein.de>

* improve backwards-compatibility
add optional arg to allow custom update-center.json targets

* pep8 fixes

* fix inconsistency in argument documentation

* Apply suggestions from code review

Co-authored-by: Amin Vakil <info@aminvakil.com>

* add unit tests

* fix pep8

* Apply suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>
Co-authored-by: Amin Vakil <info@aminvakil.com>

changelogs/fragments/1334-jenkins-plugin-fallback-urls.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+  - jenkins_plugin - add fallback url(s) for failure of plugin installation/download (https://github.com/ansible-collections/community.general/pull/1334).

plugins/modules/web_infrastructure/jenkins_plugin.py

@@ -66,12 +66,33 @@ options:
         C(latest) is specified.
     default: 86400
   updates_url:
-    type: str
+    type: list
+    elements: str
     description:
-      - URL of the Update Centre.
+      - A list of base URL(s) to retrieve I(update-center.json), and direct plugin files from.
-      - Used as the base URL to download the plugins and the
+      - This can be a list since community.general 3.3.0.
-        I(update-center.json) JSON file.
+    default: ['https://updates.jenkins.io', 'http://mirrors.jenkins.io']
-    default: https://updates.jenkins.io
+  update_json_url_segment:
+    type: list
+    elements: str
+    description:
+      - A list of URL segment(s) to retrieve the update center json file from.
+    default: ['update-center.json', 'updates/update-center.json']
+    version_added: 3.3.0
+  latest_plugins_url_segments:
+    type: list
+    elements: str
+    description:
+      - Path inside the I(updates_url) to get latest plugins from.
+    default: ['latest']
+    version_added: 3.3.0
+  versioned_plugins_url_segments:
+    type: list
+    elements: str
+    description:
+      - Path inside the I(updates_url) to get specific version of plugins from.
+    default: ['download/plugins', 'plugins']
+    version_added: 3.3.0
   url:
     type: str
     description:
@@ -283,6 +304,10 @@ import tempfile
 import time
 
 
+class FailedInstallingWithPluginManager(Exception):
+    pass
+
+
 class JenkinsPlugin(object):
     def __init__(self, module):
         # To be able to call fail_json
@@ -330,9 +355,42 @@ class JenkinsPlugin(object):
 
         return json_data
 
+    def _get_urls_data(self, urls, what=None, msg_status=None, msg_exception=None, **kwargs):
+        # Compose default messages
+        if msg_status is None:
+            msg_status = "Cannot get %s" % what
+
+        if msg_exception is None:
+            msg_exception = "Retrieval of %s failed." % what
+
+        errors = {}
+        for url in urls:
+            err_msg = None
+            try:
+                self.module.debug("fetching url: %s" % url)
+                response, info = fetch_url(
+                    self.module, url, timeout=self.timeout, cookies=self.cookies,
+                    headers=self.crumb, **kwargs)
+
+                if info['status'] == 200:
+                    return response
+                else:
+                    err_msg = ("%s. fetching url %s failed. response code: %s" % (msg_status, url, info['status']))
+                    if info['status'] > 400:  # extend error message
+                        err_msg = "%s. response body: %s" % (err_msg, info['body'])
+            except Exception as e:
+                err_msg = "%s. fetching url %s failed. error msg: %s" % (msg_status, url, to_native(e))
+            finally:
+                if err_msg is not None:
+                    self.module.debug(err_msg)
+                    errors[url] = err_msg
+
+        # failed on all urls
+        self.module.fail_json(msg=msg_exception, details=errors)
+
     def _get_url_data(
             self, url, what=None, msg_status=None, msg_exception=None,
-            **kwargs):
+            dont_fail=False, **kwargs):
         # Compose default messages
         if msg_status is None:
             msg_status = "Cannot get %s" % what
@@ -347,8 +405,14 @@ class JenkinsPlugin(object):
                 headers=self.crumb, **kwargs)
 
             if info['status'] != 200:
+                if dont_fail:
+                    raise FailedInstallingWithPluginManager(info['msg'])
+                else:
                     self.module.fail_json(msg=msg_status, details=info['msg'])
         except Exception as e:
+            if dont_fail:
+                raise FailedInstallingWithPluginManager(e)
+            else:
                 self.module.fail_json(msg=msg_exception, details=to_native(e))
 
         return response
@@ -394,14 +458,7 @@ class JenkinsPlugin(object):
 
                 break
 
-    def install(self):
+    def _install_with_plugin_manager(self):
-        changed = False
-        plugin_file = (
-            '%s/plugins/%s.jpi' % (
-                self.params['jenkins_home'],
-                self.params['name']))
-
-        if not self.is_installed and self.params['version'] in [None, 'latest']:
         if not self.module.check_mode:
             # Install the plugin (with dependencies)
             install_script = (
@@ -424,7 +481,8 @@ class JenkinsPlugin(object):
                 "%s/scriptText" % self.url,
                 msg_status="Cannot install plugin.",
                 msg_exception="Plugin installation has failed.",
-                    data=data)
+                data=data,
+                dont_fail=True)
 
             hpi_file = '%s/plugins/%s.hpi' % (
                 self.params['jenkins_home'],
@@ -433,8 +491,21 @@ class JenkinsPlugin(object):
             if os.path.isfile(hpi_file):
                 os.remove(hpi_file)
 
+    def install(self):
+        changed = False
+        plugin_file = (
+            '%s/plugins/%s.jpi' % (
+                self.params['jenkins_home'],
+                self.params['name']))
+
+        if not self.is_installed and self.params['version'] in [None, 'latest']:
+            try:
+                self._install_with_plugin_manager()
                 changed = True
-        else:
+            except FailedInstallingWithPluginManager:  # Fallback to manually downloading the plugin
+                pass
+
+        if not changed:
             # Check if the plugin directory exists
             if not os.path.isdir(self.params['jenkins_home']):
                 self.module.fail_json(
@@ -449,26 +520,17 @@ class JenkinsPlugin(object):
 
             if self.params['version'] in [None, 'latest']:
                 # Take latest version
-                plugin_url = (
+                plugin_urls = self._get_latest_plugin_urls()
-                    "%s/latest/%s.hpi" % (
-                        self.params['updates_url'],
-                        self.params['name']))
             else:
                 # Take specific version
-                plugin_url = (
+                plugin_urls = self._get_versioned_plugin_urls()
-                    "{0}/download/plugins/"
-                    "{1}/{2}/{1}.hpi".format(
-                        self.params['updates_url'],
-                        self.params['name'],
-                        self.params['version']))
-
             if (
                     self.params['updates_expiration'] == 0 or
                     self.params['version'] not in [None, 'latest'] or
                     checksum_old is None):
 
                 # Download the plugin file directly
-                r = self._download_plugin(plugin_url)
+                r = self._download_plugin(plugin_urls)
 
                 # Write downloaded plugin into file if checksums don't match
                 if checksum_old is None:
@@ -498,7 +560,7 @@ class JenkinsPlugin(object):
                 # If the latest version changed, download it
                 if checksum_old != to_bytes(plugin_data['sha1']):
                     if not self.module.check_mode:
-                        r = self._download_plugin(plugin_url)
+                        r = self._download_plugin(plugin_urls)
                         self._write_file(plugin_file, r)
 
                     changed = True
@@ -521,6 +583,27 @@ class JenkinsPlugin(object):
 
         return changed
 
+    def _get_latest_plugin_urls(self):
+        urls = []
+        for base_url in self.params['updates_url']:
+            for update_segment in self.params['latest_plugins_url_segments']:
+                urls.append("{0}/{1}/{2}.hpi".format(base_url, update_segment, self.params['name']))
+        return urls
+
+    def _get_versioned_plugin_urls(self):
+        urls = []
+        for base_url in self.params['updates_url']:
+            for versioned_segment in self.params['versioned_plugins_url_segments']:
+                urls.append("{0}/{1}/{2}/{3}/{2}.hpi".format(base_url, versioned_segment, self.params['name'], self.params['version']))
+        return urls
+
+    def _get_update_center_urls(self):
+        urls = []
+        for base_url in self.params['updates_url']:
+            for update_json in self.params['update_json_url_segment']:
+                urls.append("{0}/{1}".format(base_url, update_json))
+        return urls
+
     def _download_updates(self):
         updates_filename = 'jenkins-plugin-cache.json'
         updates_dir = os.path.expanduser('~/.ansible/tmp')
@@ -540,11 +623,11 @@ class JenkinsPlugin(object):
 
         # Download the updates file if needed
         if download_updates:
-            url = "%s/update-center.json" % self.params['updates_url']
+            urls = self._get_update_center_urls()
 
             # Get the data
-            r = self._get_url_data(
+            r = self._get_urls_data(
-                url,
+                urls,
                 msg_status="Remote updates not found.",
                 msg_exception="Updates download failed.")
 
@@ -602,15 +685,14 @@ class JenkinsPlugin(object):
 
         return data['plugins'][self.params['name']]
 
-    def _download_plugin(self, plugin_url):
+    def _download_plugin(self, plugin_urls):
         # Download the plugin
-        r = self._get_url_data(
+
-            plugin_url,
+        return self._get_urls_data(
+            plugin_urls,
             msg_status="Plugin not found.",
             msg_exception="Plugin download failed.")
 
-        return r
-
     def _write_file(self, f, data):
         # Store the plugin into a temp file and then move it
         tmp_f_fd, tmp_f = tempfile.mkstemp()
@@ -721,7 +803,12 @@ def main():
             default='present'),
         timeout=dict(default=30, type="int"),
         updates_expiration=dict(default=86400, type="int"),
-        updates_url=dict(default='https://updates.jenkins.io'),
+        updates_url=dict(type="list", elements="str", default=['https://updates.jenkins.io',
+                                                               'http://mirrors.jenkins.io']),
+        update_json_url_segment=dict(type="list", elements="str", default=['update-center.json',
+                                                                           'updates/update-center.json']),
+        latest_plugins_url_segments=dict(type="list", elements="str", default=['latest']),
+        versioned_plugins_url_segments=dict(type="list", elements="str", default=['download/plugins', 'plugins']),
         url=dict(default='http://localhost:8080'),
         url_password=dict(no_log=True),
         version=dict(),

tests/unit/plugins/modules/web_infrastructure/test_jenkins_plugin.py

@@ -151,3 +151,40 @@ def test__get_json_data(mocker):
         'CSRF')
 
     assert isinstance(json_data, Mapping)
+
+
+def test__new_fallback_urls(mocker):
+    "test generation of new fallback URLs"
+
+    params = {
+        "url": "http://fake.jenkins.server",
+        "timeout": 30,
+        "name": "test-plugin",
+        "version": "1.2.3",
+        "updates_url": ["https://some.base.url"],
+        "latest_plugins_url_segments": ["test_latest"],
+        "versioned_plugins_url_segments": ["ansible", "versioned_plugins"],
+        "update_json_url_segment": ["unreachable", "updates/update-center.json"],
+    }
+    module = mocker.Mock()
+    module.params = params
+
+    JenkinsPlugin._csrf_enabled = pass_function
+    JenkinsPlugin._get_installed_plugins = pass_function
+
+    jenkins_plugin = JenkinsPlugin(module)
+
+    latest_urls = jenkins_plugin._get_latest_plugin_urls()
+    assert isInList(latest_urls, "https://some.base.url/test_latest/test-plugin.hpi")
+    versioned_urls = jenkins_plugin._get_versioned_plugin_urls()
+    assert isInList(versioned_urls, "https://some.base.url/versioned_plugins/test-plugin/1.2.3/test-plugin.hpi")
+    json_urls = jenkins_plugin._get_update_center_urls()
+    assert isInList(json_urls, "https://some.base.url/updates/update-center.json")
+
+
+def isInList(l, i):
+    print("checking if %s in %s" % (i, l))
+    for item in l:
+        if item == i:
+            return True
+    return False