Utilize urllib3.contrib.pyopenssl functionality for SNI capability in python versions lacking SNI support

Also add SNI tests, move test_uri to destructive since we are messing with packages for SNI testing

lib/ansible/module_utils/urls.py

@@ -118,6 +118,15 @@ try:
 except ImportError:
     HAS_SSLCONTEXT = False
 
+try:
+    try:
+        from urllib3.contrib.pyopenssl import ssl_wrap_socket
+    except ImportError:
+        from requests.packages.urllib3.contrib.pyopenssl import ssl_wrap_socket
+    HAS_URLLIB3_SNI_SUPPORT = True
+except ImportError:
+    HAS_URLLIB3_SNI_SUPPORT = False
+
 # Select a protocol that includes all secure tls protocols
 # Exclude insecure ssl protocols if possible
 
@@ -340,6 +349,8 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib2, 'HTTPSHandler'):
 
             if HAS_SSLCONTEXT:
                 self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname)
+            elif HAS_URLLIB3_SNI_SUPPORT:
+                self.sock = ssl_wrap_socket(sock, keyfile=self.key_file, cert_reqs=ssl.CERT_NONE, certfile=self.cert_file, ssl_version=PROTOCOL, server_hostname=server_hostname)
             else:
                 self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL)
 
@@ -607,6 +618,8 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     self.validate_proxy_response(connect_result)
                     if context:
                         ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
+                    elif HAS_URLLIB3_SNI_SUPPORT:
+                        ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
                     else:
                         ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                         match_hostname(ssl_s.getpeercert(), self.hostname)
@@ -616,6 +629,8 @@ class SSLValidationHandler(urllib2.BaseHandler):
                 s.connect((self.hostname, self.port))
                 if context:
                     ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
+                elif HAS_URLLIB3_SNI_SUPPORT:
+                    ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
                 else:
                     ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                     match_hostname(ssl_s.getpeercert(), self.hostname)
@@ -631,13 +646,27 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     ' Make sure your managed systems have a valid CA'
                     ' certificate installed.  If the website serving the url'
                     ' uses SNI you need python >= 2.7.9 on your managed'
-                    ' machine.  You can use validate_certs=False if you do'
+                    ' machine or you can install `urllib3`, `pyopenssl`,'
+                    ' `ndg-httpsclient`, and `pyasn1` to perform SNI'
+                    ' verification in python >= 2.6.  You can use'
+                    ' validate_certs=False if you do'
                     ' not need to confirm the server\s identity but this is'
                     ' unsafe and not recommended'
                     ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked))
                 )
         except CertificateError:
-            raise SSLValidationError("SSL Certificate does not belong to %s.  Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname)
+            raise SSLValidationError('Failed to validate the SSL certificate for %s:%s.'
+                ' Make sure your managed systems have a valid CA'
+                ' certificate installed.  If the website serving the url'
+                ' uses SNI you need python >= 2.7.9 on your managed'
+                ' machine or you can install `urllib3`, `pyopenssl`,'
+                ' `ndg-httpsclient`, and `pyasn1` to perform SNI'
+                ' verification in python >= 2.6.  You can use'
+                ' validate_certs=False if you do'
+                ' not need to confirm the server\s identity but this is'
+                ' unsafe and not recommended'
+                ' Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked))
+            )
 
         try:
             # cleanup the temp file created, don't worry

test/integration/destructive.yml

@@ -20,3 +20,4 @@
     - { role: test_docker, tags: test_docker, when: ansible_distribution != "Fedora" }
     - { role: test_zypper, tags: test_zypper}
     - { role: test_zypper_repository, tags: test_zypper_repository}
+    - { role: test_uri, tags: test_uri }

test/integration/non_destructive.yml

@@ -40,7 +40,6 @@
     - { role: test_authorized_key, tags: test_authorized_key }
     - { role: test_get_url, tags: test_get_url }
     - { role: test_embedded_module, tags: test_embedded_module }
-    - { role: test_uri, tags: test_uri }
     - { role: test_add_host, tags: test_add_host }
     # Turn on test_binary when we start testing v2
     #- { role: test_binary, tags: test_binary }

test/integration/roles/test_get_url/tasks/main.yml

@@ -62,7 +62,7 @@
   assert:
     that:
       - "result.failed == true"
-      - "'Certificate does not belong to ' in result.msg"
+      - "'Failed to validate the SSL certificate' in result.msg"
       - "stat_result.stat.exists == false"
 
 - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no

test/integration/roles/test_uri/tasks/main.yml

@@ -113,7 +113,7 @@
   assert:
     that:
       - "result.failed == true"
-      - "'SSL Certificate does not belong' in result.msg"
+      - "'Failed to validate the SSL certificate' in result.msg"
       - "stat_result.stat.exists == false"
 
 - name: Clean up any cruft from the results directory
@@ -204,3 +204,78 @@
   assert:
     that:
       - 'result.allow|default("") == "HEAD, OPTIONS, GET"'
+
+# Ubuntu12.04 doesn't have python-urllib3, this makes handling required dependencies a pain across all variations
+# We'll use this to just skip 12.04 on those tests.  We should be sufficiently covered with other OSes and versions
+- name: Set fact if running on Ubuntu 12.04
+  set_fact:
+    is_ubuntu_precise: "{{ ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'precise' }}"
+
+- name: Test that SNI succeeds on python versions that have SNI
+  uri:
+    url: 'https://sni.velox.ch'
+    return_content: true
+  when: ansible_python.has_sslcontext
+  register: result
+
+- name: Assert SNI verification succeeds on new python
+  assert:
+    that:
+      - result|success
+      - '"Great! Your client" in result.content'
+  when: ansible_python.has_sslcontext
+
+- name: Verify SNI verification fails on old python without urllib3 contrib
+  uri:
+    url: 'https://sni.velox.ch'
+  ignore_errors: true
+  when: not ansible_python.has_sslcontext
+  register: result
+
+- name: Assert SNI verification fails on old python
+  assert:
+    that:
+      - result|failed
+  when: not result|skipped
+
+- name: install OS packages that are needed for SNI on old python
+  package:
+    name: "{{ item }}"
+  with_items: "{{ uri_os_packages[ansible_os_family] }}"
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: install python modules for Older Python SNI verification
+  pip:
+    name: "{{ item }}"
+  with_items:
+    - ndg-httpsclient
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: Verify SNI verificaiton succeeds on old python with urllib3 contrib
+  uri:
+    url: 'https://sni.velox.ch'
+    return_content: true
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+  register: result
+
+- name: Assert SNI verification succeeds on old python
+  assert:
+    that:
+      - result|success
+      - '"Great! Your client" in result.content'
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: Uninstall ndg-httpsclient and urllib3
+  pip:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - ndg-httpsclient
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
+
+- name: uninstall OS packages that are needed for SNI on old python
+  package:
+    name: "{{ item }}"
+    state: absent
+  with_items: "{{ uri_os_packages[ansible_os_family] }}"
+  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool

test/integration/roles/test_uri/vars/main.yml

@@ -0,0 +1,9 @@
+uri_os_packages:
+  RedHat:
+    - python-pyasn1
+    - pyOpenSSL
+    - python-urllib3
+  Debian:
+    - python-pyasn1
+    - python-openssl
+    - python-urllib3