Add requests.Session like class (#37622)

* Adds requests.Session like class

* py2 syntax fix

* Add a few examples to the Request docstrings

* Add helper methods and docs

* Fix test failures

* Switch tests to test Request instead of open_url, add simple open_url test to validate funcitonality

* Fix filename in replace-urlopen code smell test

lib/ansible/module_utils/urls.py

@@ -816,16 +816,114 @@ def maybe_add_ssl_handler(url, validate_certs):
         return SSLValidationHandler(hostname, port)
 
 
-def open_url(url, data=None, headers=None, method=None, use_proxy=True,
+class Request:
-             force=False, last_mod_time=None, timeout=10, validate_certs=True,
+    def __init__(self, headers=None, use_proxy=True, force=False, timeout=10, validate_certs=True,
+                 url_username=None, url_password=None, http_agent=None, force_basic_auth=False,
+                 follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None):
+        """This class works somewhat similarly to the ``Session`` class of from requests
+        by defining a cookiejar that an be used across requests as well as cascaded defaults that
+        can apply to repeated requests
+
+        For documentation of params, see ``Request.open``
+
+        >>> from ansible.module_utils.urls import Request
+        >>> r = Request()
+        >>> r.open('GET', 'http://httpbin.org/cookies/set?k1=v1').read()
+        '{\n  "cookies": {\n    "k1": "v1"\n  }\n}\n'
+        >>> r = Request(url_username='user', url_password='passwd')
+        >>> r.open('GET', 'http://httpbin.org/basic-auth/user/passwd').read()
+        '{\n  "authenticated": true, \n  "user": "user"\n}\n'
+        >>> r = Request(headers=dict(foo='bar'))
+        >>> r.open('GET', 'http://httpbin.org/get', headers=dict(baz='qux')).read()
+
+        """
+
+        self.headers = headers or {}
+        if not isinstance(self.headers, dict):
+            raise ValueError("headers must be a dict")
+        self.use_proxy = use_proxy
+        self.force = force
+        self.timeout = timeout
+        self.validate_certs = validate_certs
+        self.url_username = url_username
+        self.url_password = url_password
+        self.http_agent = http_agent
+        self.force_basic_auth = force_basic_auth
+        self.follow_redirects = follow_redirects
+        self.client_cert = client_cert
+        self.client_key = client_key
+        if isinstance(cookies, cookiejar.CookieJar):
+            self.cookies = cookies
+        else:
+            self.cookies = cookiejar.CookieJar()
+
+    def _fallback(self, value, fallback):
+        if value is None:
+            return fallback
+        return value
+
+    def open(self, method, url, data=None, headers=None, use_proxy=None,
+             force=None, last_mod_time=None, timeout=None, validate_certs=None,
              url_username=None, url_password=None, http_agent=None,
-             force_basic_auth=False, follow_redirects='urllib2',
+             force_basic_auth=None, follow_redirects=None,
              client_cert=None, client_key=None, cookies=None):
-    '''
+        """
         Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
 
         Does not require the module environment
-    '''
+
+        Returns :class:`HTTPResponse` object.
+
+        :arg method: method for the request
+        :arg url: URL to request
+
+        :kwarg data: (optional) bytes, or file-like object to send
+            in the body of the request
+        :kwarg headers: (optional) Dictionary of HTTP Headers to send with the
+            request
+        :kwarg use_proxy: (optional) Boolean of whether or not to use proxy
+        :kwarg force: (optional) Boolean of whether or not to set `cache-control: no-cache` header
+        :kwarg last_mod_time: (optional) Datetime object to use when setting If-Modified-Since header
+        :kwarg timeout: (optional) How long to wait for the server to send
+            data before giving up, as a float
+        :kwarg validate_certs: (optional) Booleani that controls whether we verify
+            the server's TLS certificate
+        :kwarg url_username: (optional) String of the user to use when authenticating
+        :kwarg url_password: (optional) String of the password to use when authenticating
+        :kwarg http_agent: (optional) String of the User-Agent to use in the request
+        :kwarg force_basic_auth: (optional) Boolean determining if auth header should be sent in the initial request
+        :kwarg follow_redirects: (optional) String of urllib2, all/yes, safe, none to determine how redirects are
+            followed, see RedirectHandlerFactory for more information
+        :kwarg client_cert: (optional) PEM formatted certificate chain file to be used for SSL client authentication.
+            This file can also include the key as well, and if the key is included, client_key is not required
+        :kwarg client_key: (optional) PEM formatted file that contains your private key to be used for SSL client
+            authentication. If client_cert contains both the certificate and key, this option is not required
+        :kwarg cookies: (optional) CookieJar object to send with the
+            request
+        :returns: HTTPResponse
+        """
+
+        method = method.upper()
+
+        if headers is None:
+            headers = {}
+        elif not isinstance(headers, dict):
+            raise ValueError("headers must be a dict")
+        headers = dict(self.headers, **headers)
+
+        use_proxy = self._fallback(use_proxy, self.use_proxy)
+        force = self._fallback(force, self.force)
+        timeout = self._fallback(timeout, self.timeout)
+        validate_certs = self._fallback(validate_certs, self.validate_certs)
+        url_username = self._fallback(url_username, self.url_username)
+        url_password = self._fallback(url_password, self.url_password)
+        http_agent = self._fallback(http_agent, self.http_agent)
+        force_basic_auth = self._fallback(force_basic_auth, self.force_basic_auth)
+        follow_redirects = self._fallback(follow_redirects, self.follow_redirects)
+        client_cert = self._fallback(client_cert, self.client_cert)
+        client_key = self._fallback(client_key, self.client_key)
+        cookies = self._fallback(cookies, self.cookies)
+
         handlers = []
         ssl_handler = maybe_add_ssl_handler(url, validate_certs)
         if ssl_handler:
@@ -835,9 +933,6 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
         if parsed.scheme != 'ftp':
             username = url_username
 
-        if headers is None:
-            headers = {}
-
             if username:
                 password = url_password
                 netloc = parsed.netloc
@@ -921,12 +1016,9 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
         urllib_request.install_opener(opener)
 
         data = to_bytes(data, nonstring='passthru')
-    if method:
+        if method not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'PATCH'):
-        if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'PATCH'):
+            raise ConnectionError('invalid HTTP request method; %s' % method)
-            raise ConnectionError('invalid HTTP request method; %s' % method.upper())
+        request = RequestWithMethod(url, method, data)
-        request = RequestWithMethod(url, method.upper(), data)
-    else:
-        request = urllib_request.Request(url, data)
 
         # add the custom agent header, to help prevent issues
         # with sites that block the default urllib agent string
@@ -943,9 +1035,6 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
             request.add_header('If-Modified-Since', tstamp)
 
         # user defined headers now, which may override things we've set above
-    if headers:
-        if not isinstance(headers, dict):
-            raise ValueError("headers provided to open_url() must be a dict")
         for header in headers:
             request.add_header(header, headers[header])
 
@@ -958,6 +1047,98 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
         r = urllib_request.urlopen(*urlopen_args)
         return r
 
+    def get(self, url, **kwargs):
+        r"""Sends a GET request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request
+        :kwarg \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('GET', url, **kwargs)
+
+    def options(self, url, **kwargs):
+        r"""Sends a OPTIONS request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request
+        :kwarg \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('OPTIONS', url, **kwargs)
+
+    def head(self, url, **kwargs):
+        r"""Sends a HEAD request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request
+        :kwarg \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('HEAD', url, **kwargs)
+
+    def post(self, url, data=None, **kwargs):
+        r"""Sends a POST request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request.
+        :kwarg data: (optional) bytes, or file-like object to send in the body of the request.
+        :kwarg \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('POST', url, data=data, **kwargs)
+
+    def put(self, url, data=None, **kwargs):
+        r"""Sends a PUT request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request.
+        :kwarg data: (optional) bytes, or file-like object to send in the body of the request.
+        :kwarg \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('PUT', url, data=data, **kwargs)
+
+    def patch(self, url, data=None, **kwargs):
+        r"""Sends a PATCH request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request.
+        :kwarg data: (optional) bytes, or file-like object to send in the body of the request.
+        :kwarg \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('PATCH', url, data=data, **kwargs)
+
+    def delete(self, url, **kwargs):
+        r"""Sends a DELETE request. Returns :class:`HTTPResponse` object.
+
+        :arg url: URL to request
+        :kwargs \*\*kwargs: Optional arguments that ``open`` takes.
+        :returns: HTTPResponse
+        """
+
+        return self.open('DELETE', url, **kwargs)
+
+
+def open_url(url, data=None, headers=None, method=None, use_proxy=True,
+             force=False, last_mod_time=None, timeout=10, validate_certs=True,
+             url_username=None, url_password=None, http_agent=None,
+             force_basic_auth=False, follow_redirects='urllib2',
+             client_cert=None, client_key=None, cookies=None):
+    '''
+    Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
+
+    Does not require the module environment
+    '''
+    method = method or 'GET'
+    return Request().open(method, url, data=data, headers=headers, use_proxy=use_proxy,
+                          force=force, last_mod_time=last_mod_time, timeout=timeout, validate_certs=validate_certs,
+                          url_username=url_username, url_password=url_password, http_agent=http_agent,
+                          force_basic_auth=force_basic_auth, follow_redirects=follow_redirects,
+                          client_cert=client_cert, client_key=client_key, cookies=cookies)
+
+
 #
 # Module-related functions
 #

test/sanity/code-smell/replace-urlopen.py

@@ -10,7 +10,7 @@ def main():
         'test/sanity/code-smell/%s' % os.path.basename(__file__),
         'lib/ansible/module_utils/six/__init__.py',
         'lib/ansible/module_utils/urls.py',
-        'test/units/module_utils/urls/test_open_url.py',
+        'test/units/module_utils/urls/test_Request.py',
         'test/units/module_utils/urls/test_fetch_url.py',
     ])
 

test/units/module_utils/urls/test_Request.py

@@ -8,10 +8,11 @@ __metaclass__ = type
 import datetime
 import os
 
-from ansible.module_utils.urls import open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod
+from ansible.module_utils.urls import Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod
 from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory
 
 import pytest
+from mock import call
 
 
 if HAS_SSLCONTEXT:
@@ -28,8 +29,62 @@ def install_opener_mock(mocker):
     return mocker.patch('ansible.module_utils.urls.urllib_request.install_opener')
 
 
-def test_open_url(urlopen_mock, install_opener_mock):
+def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
-    r = open_url('https://ansible.com/')
+    cookies = cookiejar.CookieJar()
+    request = Request(
+        headers={'foo': 'bar'},
+        use_proxy=False,
+        force=True,
+        timeout=100,
+        validate_certs=False,
+        url_username='user',
+        url_password='passwd',
+        http_agent='ansible-tests',
+        force_basic_auth=True,
+        follow_redirects='all',
+        client_cert='/tmp/client.pem',
+        client_key='/tmp/client.key',
+        cookies=cookies,
+    )
+    fallback_mock = mocker.spy(request, '_fallback')
+
+    r = request.open('GET', 'https://ansible.com')
+
+    calls = [
+        call(None, False),  # use_proxy
+        call(None, True),  # force
+        call(None, 100),  # timeout
+        call(None, False),  # validate_certs
+        call(None, 'user'),  # url_username
+        call(None, 'passwd'),  # url_password
+        call(None, 'ansible-tests'),  # http_agent
+        call(None, True),  # force_basic_auth
+        call(None, 'all'),  # follow_redirects
+        call(None, '/tmp/client.pem'),  # client_cert
+        call(None, '/tmp/client.key'),  # client_key
+        call(None, cookies),  # cookies
+    ]
+    fallback_mock.assert_has_calls(calls)
+
+    assert fallback_mock.call_count == 12  # All but headers use fallback
+
+    args = urlopen_mock.call_args[0]
+    assert args[1] is None  # data, this is handled in the Request not urlopen
+    assert args[2] == 100  # timeout
+
+    req = args[0]
+    assert req.headers == {
+        'Authorization': b'Basic dXNlcjpwYXNzd2Q=',
+        'Cache-control': 'no-cache',
+        'Foo': 'bar',
+        'User-agent': 'ansible-tests'
+    }
+    assert req.data is None
+    assert req.get_method() == 'GET'
+
+
+def test_Request_open(urlopen_mock, install_opener_mock):
+    r = Request().open('GET', 'https://ansible.com/')
     args = urlopen_mock.call_args[0]
     assert args[1] is None  # data, this is handled in the Request not urlopen
     assert args[2] == 10  # timeout
@@ -55,8 +110,8 @@ def test_open_url(urlopen_mock, install_opener_mock):
     assert len(found_handlers) == 2
 
 
-def test_open_url_http(urlopen_mock, install_opener_mock):
+def test_Request_open_http(urlopen_mock, install_opener_mock):
-    r = open_url('http://ansible.com/')
+    r = Request().open('GET', 'http://ansible.com/')
     args = urlopen_mock.call_args[0]
 
     opener = install_opener_mock.call_args[0][0]
@@ -70,22 +125,22 @@ def test_open_url_http(urlopen_mock, install_opener_mock):
     assert len(found_handlers) == 0
 
 
-def test_open_url_ftp(urlopen_mock, install_opener_mock, mocker):
+def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker):
     mocker.patch('ansible.module_utils.urls.ParseResultDottedDict.as_list', side_effect=AssertionError)
 
     # Using ftp scheme should prevent the AssertionError side effect to fire
-    r = open_url('ftp://foo@ansible.com/')
+    r = Request().open('GET', 'ftp://foo@ansible.com/')
 
 
-def test_open_url_headers(urlopen_mock, install_opener_mock):
+def test_Request_open_headers(urlopen_mock, install_opener_mock):
-    r = open_url('http://ansible.com/', headers={'Foo': 'bar'})
+    r = Request().open('GET', 'http://ansible.com/', headers={'Foo': 'bar'})
     args = urlopen_mock.call_args[0]
     req = args[0]
     assert req.headers == {'Foo': 'bar'}
 
 
-def test_open_url_username(urlopen_mock, install_opener_mock):
+def test_Request_open_username(urlopen_mock, install_opener_mock):
-    r = open_url('http://ansible.com/', url_username='user')
+    r = Request().open('GET', 'http://ansible.com/', url_username='user')
 
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
@@ -103,8 +158,8 @@ def test_open_url_username(urlopen_mock, install_opener_mock):
     assert found_handlers[0].passwd.passwd[None] == {(('ansible.com', '/'),): ('user', None)}
 
 
-def test_open_url_username_in_url(urlopen_mock, install_opener_mock):
+def test_Request_open_username_in_url(urlopen_mock, install_opener_mock):
-    r = open_url('http://user2@ansible.com/')
+    r = Request().open('GET', 'http://user2@ansible.com/')
 
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
@@ -121,8 +176,8 @@ def test_open_url_username_in_url(urlopen_mock, install_opener_mock):
     assert found_handlers[0].passwd.passwd[None] == {(('ansible.com', '/'),): ('user2', '')}
 
 
-def test_open_url_username_force_basic(urlopen_mock, install_opener_mock):
+def test_Request_open_username_force_basic(urlopen_mock, install_opener_mock):
-    r = open_url('http://ansible.com/', url_username='user', url_password='passwd', force_basic_auth=True)
+    r = Request().open('GET', 'http://ansible.com/', url_username='user', url_password='passwd', force_basic_auth=True)
 
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
@@ -144,8 +199,8 @@ def test_open_url_username_force_basic(urlopen_mock, install_opener_mock):
     assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q='
 
 
-def test_open_url_auth_in_netloc(urlopen_mock, install_opener_mock):
+def test_Request_open_auth_in_netloc(urlopen_mock, install_opener_mock):
-    r = open_url('http://user:passwd@ansible.com/')
+    r = Request().open('GET', 'http://user:passwd@ansible.com/')
     args = urlopen_mock.call_args[0]
     req = args[0]
     assert req.get_full_url() == 'http://ansible.com/'
@@ -166,31 +221,31 @@ def test_open_url_auth_in_netloc(urlopen_mock, install_opener_mock):
     assert len(found_handlers) == 2
 
 
-def test_open_url_netrc(urlopen_mock, install_opener_mock, monkeypatch):
+def test_Request_open_netrc(urlopen_mock, install_opener_mock, monkeypatch):
     here = os.path.dirname(__file__)
 
     monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc'))
-    r = open_url('http://ansible.com/')
+    r = Request().open('GET', 'http://ansible.com/')
     args = urlopen_mock.call_args[0]
     req = args[0]
     assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q='
 
-    r = open_url('http://foo.ansible.com/')
+    r = Request().open('GET', 'http://foo.ansible.com/')
     args = urlopen_mock.call_args[0]
     req = args[0]
     assert 'Authorization' not in req.headers
 
     monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc.nonexistant'))
-    r = open_url('http://ansible.com/')
+    r = Request().open('GET', 'http://ansible.com/')
     args = urlopen_mock.call_args[0]
     req = args[0]
     assert 'Authorization' not in req.headers
 
 
-def test_open_url_no_proxy(urlopen_mock, install_opener_mock, mocker):
+def test_Request_open_no_proxy(urlopen_mock, install_opener_mock, mocker):
     build_opener_mock = mocker.patch('ansible.module_utils.urls.urllib_request.build_opener')
 
-    r = open_url('http://ansible.com/', use_proxy=False)
+    r = Request().open('GET', 'http://ansible.com/', use_proxy=False)
 
     handlers = build_opener_mock.call_args[0]
     found_handlers = []
@@ -202,8 +257,8 @@ def test_open_url_no_proxy(urlopen_mock, install_opener_mock, mocker):
 
 
 @pytest.mark.skipif(not HAS_SSLCONTEXT, reason="requires SSLContext")
-def test_open_url_no_validate_certs(urlopen_mock, install_opener_mock):
+def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
-    r = open_url('https://ansible.com/', validate_certs=False)
+    r = Request().open('GET', 'https://ansible.com/', validate_certs=False)
 
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
@@ -224,13 +279,13 @@ def test_open_url_no_validate_certs(urlopen_mock, install_opener_mock):
     assert context.check_hostname is False
 
 
-def test_open_url_client_cert(urlopen_mock, install_opener_mock):
+def test_Request_open_client_cert(urlopen_mock, install_opener_mock):
     here = os.path.dirname(__file__)
 
     client_cert = os.path.join(here, 'fixtures/client.pem')
     client_key = os.path.join(here, 'fixtures/client.key')
 
-    r = open_url('https://ansible.com/', client_cert=client_cert, client_key=client_key)
+    r = Request().open('GET', 'https://ansible.com/', client_cert=client_cert, client_key=client_key)
 
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
@@ -252,8 +307,8 @@ def test_open_url_client_cert(urlopen_mock, install_opener_mock):
     assert https_connection.cert_file == client_cert
 
 
-def test_open_url_cookies(urlopen_mock, install_opener_mock):
+def test_Request_open_cookies(urlopen_mock, install_opener_mock):
-    r = open_url('https://ansible.com/', cookies=cookiejar.CookieJar())
+    r = Request().open('GET', 'https://ansible.com/', cookies=cookiejar.CookieJar())
 
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
@@ -267,13 +322,13 @@ def test_open_url_cookies(urlopen_mock, install_opener_mock):
     assert cookies_handler is not None
 
 
-def test_open_url_invalid_method(urlopen_mock, install_opener_mock):
+def test_Request_open_invalid_method(urlopen_mock, install_opener_mock):
     with pytest.raises(ConnectionError):
-        r = open_url('https://ansible.com/', method='BOGUS')
+        r = Request().open('BOGUS', 'https://ansible.com/')
 
 
-def test_open_url_custom_method(urlopen_mock, install_opener_mock):
+def test_Request_open_custom_method(urlopen_mock, install_opener_mock):
-    r = open_url('https://ansible.com/', method='DELETE')
+    r = Request().open('DELETE', 'https://ansible.com/')
 
     args = urlopen_mock.call_args[0]
     req = args[0]
@@ -281,8 +336,8 @@ def test_open_url_custom_method(urlopen_mock, install_opener_mock):
     assert isinstance(req, RequestWithMethod)
 
 
-def test_open_url_user_agent(urlopen_mock, install_opener_mock):
+def test_Request_open_user_agent(urlopen_mock, install_opener_mock):
-    r = open_url('https://ansible.com/', http_agent='ansible-tests')
+    r = Request().open('GET', 'https://ansible.com/', http_agent='ansible-tests')
 
     args = urlopen_mock.call_args[0]
     req = args[0]
@@ -290,8 +345,8 @@ def test_open_url_user_agent(urlopen_mock, install_opener_mock):
     assert req.headers.get('User-agent') == 'ansible-tests'
 
 
-def test_open_url_force(urlopen_mock, install_opener_mock):
+def test_Request_open_force(urlopen_mock, install_opener_mock):
-    r = open_url('https://ansible.com/', force=True, last_mod_time=datetime.datetime.now())
+    r = Request().open('GET', 'https://ansible.com/', force=True, last_mod_time=datetime.datetime.now())
 
     args = urlopen_mock.call_args[0]
     req = args[0]
@@ -300,9 +355,9 @@ def test_open_url_force(urlopen_mock, install_opener_mock):
     assert 'If-modified-since' not in req.headers
 
 
-def test_open_url_last_mod(urlopen_mock, install_opener_mock):
+def test_Request_open_last_mod(urlopen_mock, install_opener_mock):
     now = datetime.datetime.now()
-    r = open_url('https://ansible.com/', last_mod_time=now)
+    r = Request().open('GET', 'https://ansible.com/', last_mod_time=now)
 
     args = urlopen_mock.call_args[0]
     req = args[0]
@@ -310,6 +365,38 @@ def test_open_url_last_mod(urlopen_mock, install_opener_mock):
     assert req.headers.get('If-modified-since') == now.strftime('%a, %d %b %Y %H:%M:%S +0000')
 
 
-def test_open_url_headers_not_dict(urlopen_mock, install_opener_mock):
+def test_Request_open_headers_not_dict(urlopen_mock, install_opener_mock):
     with pytest.raises(ValueError):
-        r = open_url('https://ansible.com/', headers=['bob'])
+        Request().open('GET', 'https://ansible.com/', headers=['bob'])
+
+
+def test_Request_init_headers_not_dict(urlopen_mock, install_opener_mock):
+    with pytest.raises(ValueError):
+        Request(headers=['bob'])
+
+
+@pytest.mark.parametrize('method,kwargs', [
+    ('get', {}),
+    ('options', {}),
+    ('head', {}),
+    ('post', {'data': None}),
+    ('put', {'data': None}),
+    ('patch', {'data': None}),
+    ('delete', {}),
+])
+def test_methods(method, kwargs, mocker):
+    expected = method.upper()
+    open_mock = mocker.patch('ansible.module_utils.urls.Request.open')
+    request = Request()
+    getattr(request, method)('https://ansible.com')
+    open_mock.assert_called_once_with(expected, 'https://ansible.com', **kwargs)
+
+
+def test_open_url(urlopen_mock, install_opener_mock, mocker):
+    req_mock = mocker.patch('ansible.module_utils.urls.Request.open')
+    open_url('https://ansible.com/')
+    req_mock.assert_called_once_with('GET', 'https://ansible.com/', data=None, headers=None, use_proxy=True,
+                                     force=False, last_mod_time=None, timeout=10, validate_certs=True,
+                                     url_username=None, url_password=None, http_agent=None,
+                                     force_basic_auth=False, follow_redirects='urllib2',
+                                     client_cert=None, client_key=None, cookies=None)