From acd69bcc77b22a2ff6cc25bb2c9ddf06019609d1 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Wed, 31 Aug 2016 16:03:20 +0200 Subject: [PATCH] Fix url lookup for python 3 (#17295) * Use six instead of urllib2, for python 3 compat * Open the certificate file using binary mode On python3, os.write requires 'bytes'. Also avoid using a too broad exception, since the issue was hard to spot due to it. * Do not add the header User-agent if not set Python3 module do raise a exception if a header is not a string-like object, and the default value is None. --- lib/ansible/module_utils/urls.py | 10 ++++++---- lib/ansible/plugins/lookup/url.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 2ed8e10f15..9d8422f3a7 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -115,6 +115,7 @@ except ImportError: import ansible.module_utils.six.moves.urllib.request as urllib_request import ansible.module_utils.six.moves.urllib.error as urllib_error +from ansible.module_utils.six import b try: # python3 @@ -606,11 +607,11 @@ class SSLValidationHandler(urllib_request.BaseHandler): full_path = os.path.join(path, f) if os.path.isfile(full_path) and os.path.splitext(f)[1] in ('.crt','.pem'): try: - cert_file = open(full_path, 'r') + cert_file = open(full_path, 'rb') os.write(tmp_fd, cert_file.read()) - os.write(tmp_fd, '\n') + os.write(tmp_fd, b('\n')) cert_file.close() - except: + except (OSError, IOError): pass return (tmp_path, paths_checked) @@ -844,7 +845,8 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True, # add the custom agent header, to help prevent issues # with sites that block the default urllib agent string - request.add_header('User-agent', http_agent) + if http_agent: + request.add_header('User-agent', http_agent) # if we're ok with getting a 304, set the timestamp in the # header, otherwise make sure we don't get a cached copy diff --git a/lib/ansible/plugins/lookup/url.py b/lib/ansible/plugins/lookup/url.py index 9f643bc420..be9d2e08b0 100644 --- a/lib/ansible/plugins/lookup/url.py +++ b/lib/ansible/plugins/lookup/url.py @@ -17,12 +17,12 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import urllib2 from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError from ansible.utils.unicode import to_unicode +from ansible.compat.six.moves.urllib.error import HTTPError, URLError try: from __main__ import display @@ -42,9 +42,9 @@ class LookupModule(LookupBase): display.vvvv("url lookup connecting to %s" % term) try: response = open_url(term, validate_certs=validate_certs) - except urllib2.HTTPError as e: + except HTTPError as e: raise AnsibleError("Received HTTP error for %s : %s" % (term, str(e))) - except urllib2.URLError as e: + except URLError as e: raise AnsibleError("Failed lookup url for %s : %s" % (term, str(e))) except SSLValidationError as e: raise AnsibleError("Error validating the server's certificate for %s: %s" % (term, str(e)))