Merge pull request #15289 from sivel/sni-urllib3-contrib-try2

Optional Use of urllib3 for SNI verification

lib/ansible/module_utils/urls.py

@@ -118,6 +118,15 @@ try:
 except ImportError:
     HAS_SSLCONTEXT = False
 
+try:
+    try:
+        from urllib3.contrib.pyopenssl import ssl_wrap_socket
+    except ImportError:
+        from requests.packages.urllib3.contrib.pyopenssl import ssl_wrap_socket
+    HAS_URLLIB3_SNI_SUPPORT = True
+except ImportError:
+    HAS_URLLIB3_SNI_SUPPORT = False
+
 # Select a protocol that includes all secure tls protocols
 # Exclude insecure ssl protocols if possible
 
@@ -340,6 +349,8 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib2, 'HTTPSHandler'):
 
             if HAS_SSLCONTEXT:
                 self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname)
+            elif HAS_URLLIB3_SNI_SUPPORT:
+                self.sock = ssl_wrap_socket(sock, keyfile=self.key_file, cert_reqs=ssl.CERT_NONE, certfile=self.cert_file, ssl_version=PROTOCOL, server_hostname=server_hostname)
             else:
                 self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL)
 
@@ -476,6 +487,33 @@ def RedirectHandlerFactory(follow_redirects=None, validate_certs=True):
     return RedirectHandler
 
 
+def build_ssl_validation_error(hostname, port, paths):
+    '''Inteligently build out the SSLValidationError based on what support
+    you have installed
+    '''
+
+    msg = [
+        ('Failed to validate the SSL certificate for %s:%s.'
+         ' Make sure your managed systems have a valid CA'
+         ' certificate installed.')
+    ]
+    if not HAS_SSLCONTEXT:
+        msg.append('If the website serving the url uses SNI you need'
+                   ' python >= 2.7.9 on your managed machine')
+        if not HAS_URLLIB3_SNI_SUPPORT:
+            msg.append('or you can install the `urllib3`, `pyopenssl`,'
+                       ' `ndg-httpsclient`, and `pyasn1` python modules')
+
+        msg.append('to perform SNI verification in python >= 2.6.')
+
+    msg.append('You can use validate_certs=False if you do'
+               ' not need to confirm the servers identity but this is'
+               ' unsafe and not recommended.'
+               ' Paths checked for this platform: %s')
+
+    raise SSLValidationError(' '.join(msg) % (hostname, port, ", ".join(paths)))
+
+
 class SSLValidationHandler(urllib2.BaseHandler):
     '''
     A custom handler class for SSL validation.
@@ -607,6 +645,8 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     self.validate_proxy_response(connect_result)
                     if context:
                         ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
+                    elif HAS_URLLIB3_SNI_SUPPORT:
+                        ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
                     else:
                         ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                         match_hostname(ssl_s.getpeercert(), self.hostname)
@@ -616,6 +656,8 @@ class SSLValidationHandler(urllib2.BaseHandler):
                 s.connect((self.hostname, self.port))
                 if context:
                     ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
+                elif HAS_URLLIB3_SNI_SUPPORT:
+                    ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
                 else:
                     ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                     match_hostname(ssl_s.getpeercert(), self.hostname)
@@ -627,17 +669,9 @@ class SSLValidationHandler(urllib2.BaseHandler):
             if 'connection refused' in str(e).lower():
                 raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
             else:
-                raise SSLValidationError('Failed to validate the SSL certificate for %s:%s.'
-                    ' Make sure your managed systems have a valid CA'
-                    ' certificate installed.  If the website serving the url'
-                    ' uses SNI you need python >= 2.7.9 on your managed'
-                    ' machine.  You can use validate_certs=False if you do'
-                    ' not need to confirm the server\s identity but this is'
-                    ' unsafe and not recommended'
-                    ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked))
-                )
+                build_ssl_validation_error(self.hostname, self.port, paths_checked)
         except CertificateError:
-            raise SSLValidationError("SSL Certificate does not belong to %s.  Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname)
+            build_ssl_validation_error(self.hostname, self.port, paths_checked)
 
         try:
             # cleanup the temp file created, don't worry

test/integration/destructive.yml

@@ -20,3 +20,4 @@
     - { role: test_docker, tags: test_docker, when: ansible_distribution != "Fedora" }
     - { role: test_zypper, tags: test_zypper}
     - { role: test_zypper_repository, tags: test_zypper_repository}
+    - { role: test_uri, tags: test_uri }

test/integration/non_destructive.yml

@@ -40,7 +40,6 @@
     - { role: test_authorized_key, tags: test_authorized_key }
     - { role: test_get_url, tags: test_get_url }
     - { role: test_embedded_module, tags: test_embedded_module }
-    - { role: test_uri, tags: test_uri }
     - { role: test_add_host, tags: test_add_host }
     # Turn on test_binary when we start testing v2
     #- { role: test_binary, tags: test_binary }

test/integration/roles/test_get_url/tasks/main.yml

@@ -62,7 +62,7 @@
   assert:
     that:
       - "result.failed == true"
-      - "'Certificate does not belong to ' in result.msg"
+      - "'Failed to validate the SSL certificate' in result.msg"
       - "stat_result.stat.exists == false"
 
 - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no

test/integration/roles/test_uri/tasks/main.yml

@@ -113,7 +113,7 @@
   assert:
     that:
       - "result.failed == true"
-      - "'SSL Certificate does not belong' in result.msg"
+      - "'Failed to validate the SSL certificate' in result.msg"
       - "stat_result.stat.exists == false"
 
 - name: Clean up any cruft from the results directory
@@ -204,3 +204,78 @@
   assert:
     that:
       - 'result.allow|default("") == "HEAD, OPTIONS, GET"'
+
+# Ubuntu12.04 doesn't have python-urllib3, this makes handling required dependencies a pain across all variations
+# We'll use this to just skip 12.04 on those tests.  We should be sufficiently covered with other OSes and versions
+- name: Set fact if running on Ubuntu 12.04
+  set_fact:
+    is_ubuntu_precise: "{{ ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'precise' }}"
+
+- name: Test that SNI succeeds on python versions that have SNI
+  uri:
+    url: 'https://sni.velox.ch'
+    return_content: true
+  when: ansible_python.has_sslcontext
+  register: result
+
+- name: Assert SNI verification succeeds on new python
+  assert:
+    that:
+      - result|success
+      - '"Great! Your client" in result.content'
+  when: ansible_python.has_sslcontext
+
+- name: Verify SNI verification fails on old python without urllib3 contrib
+  uri:
+    url: 'https://sni.velox.ch'
+  ignore_errors: true
+  when: not ansible_python.has_sslcontext
+  register: result
+
+- name: Assert SNI verification fails on old python
+  assert:
+    that:
+      - result|failed
+  when: not result|skipped
+
+- name: install OS packages that are needed for SNI on old python
+  package:
+    name: "{{ item }}"
+  with_items: "{{ uri_os_packages[ansible_os_family] }}"
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: install python modules for Older Python SNI verification
+  pip:
+    name: "{{ item }}"
+  with_items:
+    - ndg-httpsclient
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: Verify SNI verificaiton succeeds on old python with urllib3 contrib
+  uri:
+    url: 'https://sni.velox.ch'
+    return_content: true
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+  register: result
+
+- name: Assert SNI verification succeeds on old python
+  assert:
+    that:
+      - result|success
+      - '"Great! Your client" in result.content'
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: Uninstall ndg-httpsclient and urllib3
+  pip:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - ndg-httpsclient
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: uninstall OS packages that are needed for SNI on old python
+  package:
+    name: "{{ item }}"
+    state: absent
+  with_items: "{{ uri_os_packages[ansible_os_family] }}"
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool

test/integration/roles/test_uri/vars/main.yml

@@ -0,0 +1,9 @@
+uri_os_packages:
+  RedHat:
+    - python-pyasn1
+    - pyOpenSSL
+    - python-urllib3
+  Debian:
+    - python-pyasn1
+    - python-openssl
+    - python-urllib3