Add API pagination support to Scaleway inventory (#46117)

* Add Scaleway API pagination to server inventory call

* Move Link parsing to helper module

* Correct some PEP8 errors

* Replace AnsibleError with ScalewayException in module_utils since the former doesn't work

* Simplify the regexes to match the intended purpose

* Cleanup helper to conform to review

* Cleanup Scaleway inventory to conform to review

* Flatten the conditional branches structure

* fix a regexp typo

lib/ansible/module_utils/scaleway.py

@@ -1,4 +1,5 @@
 import json
+import re
 import sys
 
 from ansible.module_utils.basic import env_fallback
@@ -29,6 +30,29 @@ class ScalewayException(Exception):
         self.message = message
 
 
+# Specify a complete Link header, for validation purposes
+R_LINK_HEADER = r'''<[^>]+>;\srel="(first|previous|next|last)"
+    (,<[^>]+>;\srel="(first|previous|next|last)")*'''
+# Specify a single relation, for iteration and string extraction purposes
+R_RELATION = r'<(?P<target_IRI>[^>]+)>; rel="(?P<relation>first|previous|next|last)"'
+
+
+def parse_pagination_link(header):
+    if not re.match(R_LINK_HEADER, header, re.VERBOSE):
+        raise ScalewayException('Scaleway API answered with an invalid Link pagination header')
+    else:
+        relations = header.split(',')
+        parsed_relations = {}
+        rc_relation = re.compile(R_RELATION)
+        for relation in relations:
+            match = rc_relation.match(relation)
+            if not match:
+                raise ScalewayException('Scaleway API answered with an invalid relation in the Link pagination header')
+            data = match.groupdict()
+            parsed_relations[data['relation']] = data['target_IRI']
+        return parsed_relations
+
+
 class Response(object):
 
     def __init__(self, resp, info):

lib/ansible/plugins/inventory/scaleway.py

@@ -86,28 +86,40 @@ import json
 
 from ansible.errors import AnsibleError
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
-from ansible.module_utils.scaleway import SCALEWAY_LOCATION
+from ansible.module_utils.scaleway import SCALEWAY_LOCATION, parse_pagination_link
 from ansible.module_utils.urls import open_url
 from ansible.module_utils._text import to_native
 
+import ansible.module_utils.six.moves.urllib.parse as urllib_parse
+
 
 def _fetch_information(token, url):
-    try:
+    results = []
-        response = open_url(url,
+    paginated_url = url
-                            headers={'X-Auth-Token': token,
+    while True:
-                                     'Content-type': 'application/json'})
+        try:
-    except Exception as e:
+            response = open_url(paginated_url,
-        raise AnsibleError("Error while fetching %s: %s" % (url, to_native(e)))
+                                headers={'X-Auth-Token': token,
+                                         'Content-type': 'application/json'})
+        except Exception as e:
+            raise AnsibleError("Error while fetching %s: %s" % (url, to_native(e)))
+        try:
+            raw_json = json.loads(response.read())
+        except ValueError:
+            raise AnsibleError("Incorrect JSON payload")
 
-    try:
+        try:
-        raw_json = json.loads(response.read())
+            results.extend(raw_json["servers"])
-    except ValueError:
+        except KeyError:
-        raise AnsibleError("Incorrect JSON payload")
+            raise AnsibleError("Incorrect format from the Scaleway API response")
 
-    try:
+        link = response.getheader('Link')
-        return raw_json["servers"]
+        if not link:
-    except KeyError:
+            return results
-        raise AnsibleError("Incorrect format from the Scaleway API response")
+        relations = parse_pagination_link(link)
+        if 'next' not in relations:
+            return results
+        paginated_url = urllib_parse.urljoin(paginated_url, relations['next'])
 
 
 def _build_server_url(api_endpoint):