httpapi: let httpapi plugin handle HTTPErrors other than 401 (#43436)

* Hold httpapi response in BytesIO

* Let httpapi plugin deal with HTTP codes if it wants

* Python 3.5 won't json.loads() bytes

* Don't modify headers passed to send

* Move code handling back to send()

but let httpapi plugin have a say on how it happens

lib/ansible/plugins/connection/httpapi.py

@@ -142,9 +142,9 @@ options:
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_bytes
-from ansible.module_utils.six import PY3
+from ansible.module_utils.six import PY3, BytesIO
 from ansible.module_utils.six.moves import cPickle
-from ansible.module_utils.six.moves.urllib.error import HTTPError
+from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError
 from ansible.module_utils.urls import open_url
 from ansible.playbook.play_context import PlayContext
 from ansible.plugins.loader import cliconf_loader, httpapi_loader
@@ -243,7 +243,10 @@ class Connection(NetworkConnectionBase):
         )
         url_kwargs.update(kwargs)
         if self._auth:
-            url_kwargs['headers'].update(self._auth)
+            # Avoid modifying passed-in headers
+            headers = dict(kwargs.get('headers', {}))
+            headers.update(self._auth)
+            url_kwargs['headers'] = headers
         else:
             url_kwargs['url_username'] = self.get_option('remote_user')
             url_kwargs['url_password'] = self.get_option('password')
@@ -251,16 +254,20 @@ class Connection(NetworkConnectionBase):
         try:
             response = open_url(self._url + path, data=data, **url_kwargs)
         except HTTPError as exc:
-            if exc.code == 401 and self._auth:
+            is_handled = self.handle_httperror(exc)
-                # Stored auth appears to be invalid, clear and retry
+            if is_handled is True:
-                self._auth = None
-                self.login(self.get_option('remote_user'), self.get_option('password'))
                 return self.send(path, data, **kwargs)
-            raise AnsibleConnectionFailure('Could not connect to {0}: {1}'.format(self._url, exc.reason))
+            elif is_handled is False:
+                raise AnsibleConnectionFailure('Could not connect to {0}: {1}'.format(self._url + path, exc.reason))
+            else:
+                raise
+        except URLError as exc:
+            raise AnsibleConnectionFailure('Could not connect to {0}: {1}'.format(self._url + path, exc.reason))
 
-        response_text = response.read()
+        response_buffer = BytesIO()
+        response_buffer.write(response.read())
 
         # Try to assign a new auth token if one is given
-        self._auth = self.update_auth(response, response_text) or self._auth
+        self._auth = self.update_auth(response, response_buffer) or self._auth
 
-        return response, response_text
+        return response, response_buffer

lib/ansible/plugins/httpapi/__init__.py

@@ -11,6 +11,8 @@ from ansible.plugins import AnsiblePlugin
 
 class HttpApiBase(AnsiblePlugin):
     def __init__(self, connection):
+        super(HttpApiBase, self).__init__()
+
         self.connection = connection
         self._become = False
         self._become_pass = ''
@@ -49,6 +51,30 @@ class HttpApiBase(AnsiblePlugin):
 
         return None
 
+    def handle_httperror(self, exc):
+        """Overridable method for dealing with HTTP codes.
+
+        This method will attempt to handle known cases of HTTP status codes.
+        If your API uses status codes to convey information in a regular way,
+        you can override this method to handle it appropriately.
+
+        :returns:
+            * True if the code has been handled in a way that the request
+            may be resent without changes.
+            * False if this code indicates a fatal or unknown error which
+            cannot be handled by the plugin. This will result in an
+            AnsibleConnectionFailure being raised.
+            * Any other response passes the HTTPError along to the caller to
+            deal with as appropriate.
+        """
+        if exc.code == 401 and self.connection._auth:
+            # Stored auth appears to be invalid, clear and retry
+            self.connection._auth = None
+            self.login(self.connection.get_option('remote_user'), self.connection.get_option('password'))
+            return True
+
+        return False
+
     @abstractmethod
     def send_request(self, data, **message_kwargs):
         """Prepares and sends request(s) to device."""

lib/ansible/plugins/httpapi/eos.py

@@ -30,13 +30,16 @@ class HttpApi(HttpApiBase):
         request = request_builder(data, output)
         headers = {'Content-Type': 'application/json-rpc'}
 
-        response, response_text = self.connection.send('/command-api', request, headers=headers, method='POST')
+        response, response_data = self.connection.send('/command-api', request, headers=headers, method='POST')
-        try:
-            response_text = json.loads(response_text)
-        except ValueError:
-            raise ConnectionError('Response was not valid JSON, got {0}'.format(response_text))
 
-        results = handle_response(response_text)
+        try:
+            response_data = json.loads(to_text(response_data.getvalue()))
+        except ValueError:
+            raise ConnectionError('Response was not valid JSON, got {0}'.format(
+                to_text(response_data.getvalue())
+            ))
+
+        results = handle_response(response_data)
 
         if self._become:
             results = results[1:]

lib/ansible/plugins/httpapi/nxos.py

@@ -27,13 +27,16 @@ class HttpApi(HttpApiBase):
         request = request_builder(queue, output)
         headers = {'Content-Type': 'application/json'}
 
-        response, response_text = self.connection.send('/ins', request, headers=headers, method='POST')
+        response, response_data = self.connection.send('/ins', request, headers=headers, method='POST')
-        try:
-            response_text = json.loads(response_text)
-        except ValueError:
-            raise ConnectionError('Response was not valid JSON, got {0}'.format(response_text))
 
-        results = handle_response(response_text)
+        try:
+            response_data = json.loads(to_text(response_data.getvalue()))
+        except ValueError:
+            raise ConnectionError('Response was not valid JSON, got {0}'.format(
+                to_text(response_data.getvalue())
+            ))
+
+        results = handle_response(response_data)
 
         if self._become:
             results = results[1:]