From 6e9c09d7f76decbbf50ee9804da69b7c25dba9f9 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 4 Apr 2016 19:35:47 -0500 Subject: [PATCH 1/2] Utilize urllib3.contrib.pyopenssl functionality for SNI capability in python versions lacking SNI support Also add SNI tests, move test_uri to destructive since we are messing with packages for SNI testing --- lib/ansible/module_utils/urls.py | 33 +++++++- test/integration/destructive.yml | 1 + test/integration/non_destructive.yml | 1 - .../roles/test_get_url/tasks/main.yml | 2 +- .../integration/roles/test_uri/tasks/main.yml | 77 ++++++++++++++++++- test/integration/roles/test_uri/vars/main.yml | 9 +++ 6 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 test/integration/roles/test_uri/vars/main.yml diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 17c3662f5f..75a32a896e 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -118,6 +118,15 @@ try: except ImportError: HAS_SSLCONTEXT = False +try: + try: + from urllib3.contrib.pyopenssl import ssl_wrap_socket + except ImportError: + from requests.packages.urllib3.contrib.pyopenssl import ssl_wrap_socket + HAS_URLLIB3_SNI_SUPPORT = True +except ImportError: + HAS_URLLIB3_SNI_SUPPORT = False + # Select a protocol that includes all secure tls protocols # Exclude insecure ssl protocols if possible @@ -340,6 +349,8 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib2, 'HTTPSHandler'): if HAS_SSLCONTEXT: self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname) + elif HAS_URLLIB3_SNI_SUPPORT: + self.sock = ssl_wrap_socket(sock, keyfile=self.key_file, cert_reqs=ssl.CERT_NONE, certfile=self.cert_file, ssl_version=PROTOCOL, server_hostname=server_hostname) else: self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL) @@ -607,6 +618,8 @@ class SSLValidationHandler(urllib2.BaseHandler): self.validate_proxy_response(connect_result) if context: ssl_s = context.wrap_socket(s, server_hostname=self.hostname) + elif HAS_URLLIB3_SNI_SUPPORT: + ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname) else: ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL) match_hostname(ssl_s.getpeercert(), self.hostname) @@ -616,6 +629,8 @@ class SSLValidationHandler(urllib2.BaseHandler): s.connect((self.hostname, self.port)) if context: ssl_s = context.wrap_socket(s, server_hostname=self.hostname) + elif HAS_URLLIB3_SNI_SUPPORT: + ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname) else: ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL) match_hostname(ssl_s.getpeercert(), self.hostname) @@ -631,13 +646,27 @@ class SSLValidationHandler(urllib2.BaseHandler): ' Make sure your managed systems have a valid CA' ' certificate installed. If the website serving the url' ' uses SNI you need python >= 2.7.9 on your managed' - ' machine. You can use validate_certs=False if you do' + ' machine or you can install `urllib3`, `pyopenssl`,' + ' `ndg-httpsclient`, and `pyasn1` to perform SNI' + ' verification in python >= 2.6. You can use' + ' validate_certs=False if you do' ' not need to confirm the server\s identity but this is' ' unsafe and not recommended' ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked)) ) except CertificateError: - raise SSLValidationError("SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname) + raise SSLValidationError('Failed to validate the SSL certificate for %s:%s.' + ' Make sure your managed systems have a valid CA' + ' certificate installed. If the website serving the url' + ' uses SNI you need python >= 2.7.9 on your managed' + ' machine or you can install `urllib3`, `pyopenssl`,' + ' `ndg-httpsclient`, and `pyasn1` to perform SNI' + ' verification in python >= 2.6. You can use' + ' validate_certs=False if you do' + ' not need to confirm the server\s identity but this is' + ' unsafe and not recommended' + ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked)) + ) try: # cleanup the temp file created, don't worry diff --git a/test/integration/destructive.yml b/test/integration/destructive.yml index 5dd590f856..92b6e89532 100644 --- a/test/integration/destructive.yml +++ b/test/integration/destructive.yml @@ -20,3 +20,4 @@ - { role: test_docker, tags: test_docker, when: ansible_distribution != "Fedora" } - { role: test_zypper, tags: test_zypper} - { role: test_zypper_repository, tags: test_zypper_repository} + - { role: test_uri, tags: test_uri } diff --git a/test/integration/non_destructive.yml b/test/integration/non_destructive.yml index ee30fa2315..6affb7f821 100644 --- a/test/integration/non_destructive.yml +++ b/test/integration/non_destructive.yml @@ -40,7 +40,6 @@ - { role: test_authorized_key, tags: test_authorized_key } - { role: test_get_url, tags: test_get_url } - { role: test_embedded_module, tags: test_embedded_module } - - { role: test_uri, tags: test_uri } - { role: test_add_host, tags: test_add_host } # Turn on test_binary when we start testing v2 #- { role: test_binary, tags: test_binary } diff --git a/test/integration/roles/test_get_url/tasks/main.yml b/test/integration/roles/test_get_url/tasks/main.yml index 61b26f08f6..1a99304245 100644 --- a/test/integration/roles/test_get_url/tasks/main.yml +++ b/test/integration/roles/test_get_url/tasks/main.yml @@ -62,7 +62,7 @@ assert: that: - "result.failed == true" - - "'Certificate does not belong to ' in result.msg" + - "'Failed to validate the SSL certificate' in result.msg" - "stat_result.stat.exists == false" - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no diff --git a/test/integration/roles/test_uri/tasks/main.yml b/test/integration/roles/test_uri/tasks/main.yml index 4c06ea1ad7..2a5436bdaf 100644 --- a/test/integration/roles/test_uri/tasks/main.yml +++ b/test/integration/roles/test_uri/tasks/main.yml @@ -113,7 +113,7 @@ assert: that: - "result.failed == true" - - "'SSL Certificate does not belong' in result.msg" + - "'Failed to validate the SSL certificate' in result.msg" - "stat_result.stat.exists == false" - name: Clean up any cruft from the results directory @@ -204,3 +204,78 @@ assert: that: - 'result.allow|default("") == "HEAD, OPTIONS, GET"' + +# Ubuntu12.04 doesn't have python-urllib3, this makes handling required dependencies a pain across all variations +# We'll use this to just skip 12.04 on those tests. We should be sufficiently covered with other OSes and versions +- name: Set fact if running on Ubuntu 12.04 + set_fact: + is_ubuntu_precise: "{{ ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'precise' }}" + +- name: Test that SNI succeeds on python versions that have SNI + uri: + url: 'https://sni.velox.ch' + return_content: true + when: ansible_python.has_sslcontext + register: result + +- name: Assert SNI verification succeeds on new python + assert: + that: + - result|success + - '"Great! Your client" in result.content' + when: ansible_python.has_sslcontext + +- name: Verify SNI verification fails on old python without urllib3 contrib + uri: + url: 'https://sni.velox.ch' + ignore_errors: true + when: not ansible_python.has_sslcontext + register: result + +- name: Assert SNI verification fails on old python + assert: + that: + - result|failed + when: not result|skipped + +- name: install OS packages that are needed for SNI on old python + package: + name: "{{ item }}" + with_items: "{{ uri_os_packages[ansible_os_family] }}" + when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool + +- name: install python modules for Older Python SNI verification + pip: + name: "{{ item }}" + with_items: + - ndg-httpsclient + when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool + +- name: Verify SNI verificaiton succeeds on old python with urllib3 contrib + uri: + url: 'https://sni.velox.ch' + return_content: true + when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool + register: result + +- name: Assert SNI verification succeeds on old python + assert: + that: + - result|success + - '"Great! Your client" in result.content' + when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool + +- name: Uninstall ndg-httpsclient and urllib3 + pip: + name: "{{ item }}" + state: absent + with_items: + - ndg-httpsclient + when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool + +- name: uninstall OS packages that are needed for SNI on old python + package: + name: "{{ item }}" + state: absent + with_items: "{{ uri_os_packages[ansible_os_family] }}" + when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool diff --git a/test/integration/roles/test_uri/vars/main.yml b/test/integration/roles/test_uri/vars/main.yml new file mode 100644 index 0000000000..b404a14a28 --- /dev/null +++ b/test/integration/roles/test_uri/vars/main.yml @@ -0,0 +1,9 @@ +uri_os_packages: + RedHat: + - python-pyasn1 + - pyOpenSSL + - python-urllib3 + Debian: + - python-pyasn1 + - python-openssl + - python-urllib3 From 398218b6ea768464e10825b22b67b864eb31399c Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 6 Apr 2016 11:04:04 -0500 Subject: [PATCH 2/2] More intelligent building of the SSLValidationError message based on capabilities --- lib/ansible/module_utils/urls.py | 53 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 75a32a896e..7fe36d3f47 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -487,6 +487,33 @@ def RedirectHandlerFactory(follow_redirects=None, validate_certs=True): return RedirectHandler +def build_ssl_validation_error(hostname, port, paths): + '''Inteligently build out the SSLValidationError based on what support + you have installed + ''' + + msg = [ + ('Failed to validate the SSL certificate for %s:%s.' + ' Make sure your managed systems have a valid CA' + ' certificate installed.') + ] + if not HAS_SSLCONTEXT: + msg.append('If the website serving the url uses SNI you need' + ' python >= 2.7.9 on your managed machine') + if not HAS_URLLIB3_SNI_SUPPORT: + msg.append('or you can install the `urllib3`, `pyopenssl`,' + ' `ndg-httpsclient`, and `pyasn1` python modules') + + msg.append('to perform SNI verification in python >= 2.6.') + + msg.append('You can use validate_certs=False if you do' + ' not need to confirm the servers identity but this is' + ' unsafe and not recommended.' + ' Paths checked for this platform: %s') + + raise SSLValidationError(' '.join(msg) % (hostname, port, ", ".join(paths))) + + class SSLValidationHandler(urllib2.BaseHandler): ''' A custom handler class for SSL validation. @@ -642,31 +669,9 @@ class SSLValidationHandler(urllib2.BaseHandler): if 'connection refused' in str(e).lower(): raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port)) else: - raise SSLValidationError('Failed to validate the SSL certificate for %s:%s.' - ' Make sure your managed systems have a valid CA' - ' certificate installed. If the website serving the url' - ' uses SNI you need python >= 2.7.9 on your managed' - ' machine or you can install `urllib3`, `pyopenssl`,' - ' `ndg-httpsclient`, and `pyasn1` to perform SNI' - ' verification in python >= 2.6. You can use' - ' validate_certs=False if you do' - ' not need to confirm the server\s identity but this is' - ' unsafe and not recommended' - ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked)) - ) + build_ssl_validation_error(self.hostname, self.port, paths_checked) except CertificateError: - raise SSLValidationError('Failed to validate the SSL certificate for %s:%s.' - ' Make sure your managed systems have a valid CA' - ' certificate installed. If the website serving the url' - ' uses SNI you need python >= 2.7.9 on your managed' - ' machine or you can install `urllib3`, `pyopenssl`,' - ' `ndg-httpsclient`, and `pyasn1` to perform SNI' - ' verification in python >= 2.6. You can use' - ' validate_certs=False if you do' - ' not need to confirm the server\s identity but this is' - ' unsafe and not recommended' - ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked)) - ) + build_ssl_validation_error(self.hostname, self.port, paths_checked) try: # cleanup the temp file created, don't worry