diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py
index 4320ba7b8b..b4e37a22a9 100644
--- a/lib/ansible/module_utils/urls.py
+++ b/lib/ansible/module_utils/urls.py
@@ -721,15 +721,11 @@ class SSLValidationHandler(urllib_request.BaseHandler):
             # close the ssl connection
             #ssl_s.unwrap()
             s.close()
-        except (ssl.SSLError, socket.error):
-            e = get_exception()
-            # fail if we tried all of the certs but none worked
-            if 'connection refused' in str(e).lower():
-                raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
-            else:
-                build_ssl_validation_error(self.hostname, self.port, paths_checked)
-        except CertificateError:
+        except (ssl.SSLError, CertificateError):
             build_ssl_validation_error(self.hostname, self.port, paths_checked)
+        except socket.error:
+            e = get_exception()
+            raise ConnectionError('Failed to connect to %s at port %s: %s' % (self.hostname, self.port, str(e)))
 
         try:
             # cleanup the temp file created, don't worry