mirror of
https://github.com/ansible-collections/community.general.git
synced 2024-09-14 20:13:21 +02:00
Split the fetch_url() function into fetch_url and open_url().
open_url() is suitable for use outside of a module environment. Will let us use open_url to do SSL cert verification in other, non-module code.
This commit is contained in:
Toshio Kuratomi
parent
e07dde1a3c
commit
4161d78a94
1 changed files with 110 additions and 76 deletions
|
|
@ -26,12 +26,6 @@
|
|||
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
try:
|
||||
import urllib
|
||||
HAS_URLLIB = True
|
||||
except:
|
||||
HAS_URLLIB = False
|
||||
|
||||
try:
|
||||
import urllib2
|
||||
HAS_URLLIB2 = True
|
||||
|
|
@ -62,7 +56,9 @@ except ImportError:
|
|||
import httplib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import socket
|
||||
import platform
|
||||
import tempfile
|
||||
|
||||
|
||||
|
|
@ -89,6 +85,27 @@ zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg=
|
|||
-----END CERTIFICATE-----
|
||||
"""
|
||||
|
||||
#
|
||||
# Exceptions
|
||||
#
|
||||
|
||||
class ConnectionError(Exception):
|
||||
"""Failed to connect to the server"""
|
||||
pass
|
||||
|
||||
class ProxyError(ConnectionError):
|
||||
"""Failure to connect because of a proxy"""
|
||||
pass
|
||||
|
||||
class SSLValidationError(ConnectionError):
|
||||
"""Failure to connect due to SSL validation failing"""
|
||||
pass
|
||||
|
||||
class NoSSLError(SSLValidationError):
|
||||
"""Needed to connect to an HTTPS url but no ssl library available to verify the certificate"""
|
||||
pass
|
||||
|
||||
|
||||
class CustomHTTPSConnection(httplib.HTTPSConnection):
|
||||
def connect(self):
|
||||
"Connect to a host on a given (SSL) port."
|
||||
|
|
@ -153,7 +170,7 @@ def generic_urlparse(parts):
|
|||
username, password = auth.split(':', 1)
|
||||
generic_parts['username'] = username
|
||||
generic_parts['password'] = password
|
||||
generic_parts['hostname'] = hostnme
|
||||
generic_parts['hostname'] = hostname
|
||||
generic_parts['port'] = port
|
||||
except:
|
||||
generic_parts['username'] = None
|
||||
|
|
@ -189,8 +206,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
'''
|
||||
CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n"
|
||||
|
||||
def __init__(self, module, hostname, port):
|
||||
self.module = module
|
||||
def __init__(self, hostname, port):
|
||||
self.hostname = hostname
|
||||
self.port = port
|
||||
|
||||
|
|
@ -200,23 +216,22 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
|
||||
ca_certs = []
|
||||
paths_checked = []
|
||||
platform = get_platform()
|
||||
distribution = get_distribution()
|
||||
|
||||
system = platform.system()
|
||||
# build a list of paths to check for .crt/.pem files
|
||||
# based on the platform type
|
||||
paths_checked.append('/etc/ssl/certs')
|
||||
if platform == 'Linux':
|
||||
if system == 'Linux':
|
||||
paths_checked.append('/etc/pki/ca-trust/extracted/pem')
|
||||
paths_checked.append('/etc/pki/tls/certs')
|
||||
paths_checked.append('/usr/share/ca-certificates/cacert.org')
|
||||
elif platform == 'FreeBSD':
|
||||
elif system == 'FreeBSD':
|
||||
paths_checked.append('/usr/local/share/certs')
|
||||
elif platform == 'OpenBSD':
|
||||
elif system == 'OpenBSD':
|
||||
paths_checked.append('/etc/ssl')
|
||||
elif platform == 'NetBSD':
|
||||
elif system == 'NetBSD':
|
||||
ca_certs.append('/etc/openssl/certs')
|
||||
elif platform == 'SunOS':
|
||||
elif system == 'SunOS':
|
||||
paths_checked.append('/opt/local/etc/openssl/certs')
|
||||
|
||||
# fall back to a user-deployed cert in a standard
|
||||
|
|
@ -226,7 +241,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
tmp_fd, tmp_path = tempfile.mkstemp()
|
||||
|
||||
# Write the dummy ca cert if we are running on Mac OS X
|
||||
if platform == 'Darwin':
|
||||
if system == 'Darwin':
|
||||
os.write(tmp_fd, DUMMY_CA_CERT)
|
||||
# Default Homebrew path for OpenSSL certs
|
||||
paths_checked.append('/usr/local/etc/openssl')
|
||||
|
|
@ -259,7 +274,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
if int(resp_code) not in valid_codes:
|
||||
raise Exception
|
||||
except:
|
||||
self.module.fail_json(msg='Connection to proxy failed')
|
||||
raise ProxyError('Connection to proxy failed')
|
||||
|
||||
def detect_no_proxy(self, url):
|
||||
'''
|
||||
|
|
@ -304,7 +319,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
|
||||
match_hostname(ssl_s.getpeercert(), self.hostname)
|
||||
else:
|
||||
self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
|
||||
raise ProxyError('Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
|
||||
else:
|
||||
s.connect((self.hostname, self.port))
|
||||
ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
|
||||
|
|
@ -315,15 +330,14 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
except (ssl.SSLError, socket.error), e:
|
||||
# fail if we tried all of the certs but none worked
|
||||
if 'connection refused' in str(e).lower():
|
||||
self.module.fail_json(msg='Failed to connect to %s:%s.' % (self.hostname, self.port))
|
||||
raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
|
||||
else:
|
||||
self.module.fail_json(
|
||||
msg='Failed to validate the SSL certificate for %s:%s. ' % (self.hostname, self.port) + \
|
||||
'Use validate_certs=no or make sure your managed systems have a valid CA certificate installed. ' + \
|
||||
'Paths checked for this platform: %s' % ", ".join(paths_checked)
|
||||
raise SSLValidationError('Failed to validate the SSL certificate for %s:%s. '
|
||||
'Use validate_certs=False (insecure) or make sure your managed systems have a valid CA certificate installed. '
|
||||
'Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked))
|
||||
)
|
||||
except CertificateError:
|
||||
self.module.fail_json(msg="SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=no (insecure)" % self.hostname)
|
||||
raise SSLValidationError("SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname)
|
||||
|
||||
try:
|
||||
# cleanup the temp file created, don't worry
|
||||
|
|
@ -336,55 +350,23 @@ class SSLValidationHandler(urllib2.BaseHandler):
|
|||
|
||||
https_request = http_request
|
||||
|
||||
|
||||
def url_argument_spec():
|
||||
'''
|
||||
Creates an argument spec that can be used with any module
|
||||
that will be requesting content via urllib/urllib2
|
||||
'''
|
||||
return dict(
|
||||
url = dict(),
|
||||
force = dict(default='no', aliases=['thirsty'], type='bool'),
|
||||
http_agent = dict(default='ansible-httpget'),
|
||||
use_proxy = dict(default='yes', type='bool'),
|
||||
validate_certs = dict(default='yes', type='bool'),
|
||||
url_username = dict(required=False),
|
||||
url_password = dict(required=False),
|
||||
)
|
||||
|
||||
|
||||
def fetch_url(module, url, data=None, headers=None, method=None,
|
||||
use_proxy=True, force=False, last_mod_time=None, timeout=10):
|
||||
# Rewrite of fetch_url to not require the module environment
|
||||
def open_url(url, data=None, headers=None, method=None, use_proxy=True,
|
||||
force=False, last_mod_time=None, timeout=10, validate_certs=True,
|
||||
url_username=None, url_password=None, http_agent=None):
|
||||
'''
|
||||
Fetches a file from an HTTP/FTP server using urllib2
|
||||
'''
|
||||
|
||||
if not HAS_URLLIB:
|
||||
module.fail_json(msg='urllib is not installed')
|
||||
if not HAS_URLLIB2:
|
||||
module.fail_json(msg='urllib2 is not installed')
|
||||
elif not HAS_URLPARSE:
|
||||
module.fail_json(msg='urlparse is not installed')
|
||||
|
||||
r = None
|
||||
handlers = []
|
||||
info = dict(url=url)
|
||||
|
||||
distribution = get_distribution()
|
||||
# Get validate_certs from the module params
|
||||
validate_certs = module.params.get('validate_certs', True)
|
||||
|
||||
# FIXME: change the following to use the generic_urlparse function
|
||||
# to remove the indexed references for 'parsed'
|
||||
parsed = urlparse.urlparse(url)
|
||||
if parsed[0] == 'https' and validate_certs:
|
||||
if not HAS_SSL:
|
||||
if distribution == 'Redhat':
|
||||
module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended. You can also install python-ssl from EPEL')
|
||||
else:
|
||||
module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended')
|
||||
raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False, however this is unsafe and not recommended')
|
||||
if not HAS_MATCH_HOSTNAME:
|
||||
module.fail_json(msg='Available SSL validation does not check that the certificate matches the hostname. You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer. You could also use validate_certs=no, however this is unsafe and not recommended')
|
||||
raise SSLValidationError('Available SSL validation does not check that the certificate matches the hostname. You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer. You could also use validate_certs=False, however this is unsafe and not recommended')
|
||||
|
||||
# do the cert validation
|
||||
netloc = parsed[1]
|
||||
|
|
@ -398,13 +380,14 @@ def fetch_url(module, url, data=None, headers=None, method=None,
|
|||
port = 443
|
||||
# create the SSL validation handler and
|
||||
# add it to the list of handlers
|
||||
ssl_handler = SSLValidationHandler(module, hostname, port)
|
||||
ssl_handler = SSLValidationHandler(hostname, port)
|
||||
handlers.append(ssl_handler)
|
||||
|
||||
if parsed[0] != 'ftp':
|
||||
username = module.params.get('url_username', '')
|
||||
username = url_username
|
||||
|
||||
if username:
|
||||
password = module.params.get('url_password', '')
|
||||
password = url_password
|
||||
netloc = parsed[1]
|
||||
elif '@' in parsed[1]:
|
||||
credentials, netloc = parsed[1].split('@', 1)
|
||||
|
|
@ -448,14 +431,14 @@ def fetch_url(module, url, data=None, headers=None, method=None,
|
|||
|
||||
if method:
|
||||
if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'):
|
||||
module.fail_json(msg='invalid HTTP request method; %s' % method.upper())
|
||||
raise ConnectionError('invalid HTTP request method; %s' % method.upper())
|
||||
request = RequestWithMethod(url, method.upper(), data)
|
||||
else:
|
||||
request = urllib2.Request(url, data)
|
||||
|
||||
# add the custom agent header, to help prevent issues
|
||||
# with sites that block the default urllib agent string
|
||||
request.add_header('User-agent', module.params.get('http_agent'))
|
||||
request.add_header('User-agent', http_agent)
|
||||
|
||||
# if we're ok with getting a 304, set the timestamp in the
|
||||
# header, otherwise make sure we don't get a cached copy
|
||||
|
|
@ -468,20 +451,72 @@ def fetch_url(module, url, data=None, headers=None, method=None,
|
|||
# user defined headers now, which may override things we've set above
|
||||
if headers:
|
||||
if not isinstance(headers, dict):
|
||||
module.fail_json("headers provided to fetch_url() must be a dict")
|
||||
raise ValueError("headers provided to fetch_url() must be a dict")
|
||||
for header in headers:
|
||||
request.add_header(header, headers[header])
|
||||
|
||||
if sys.version_info < (2,6,0):
|
||||
# urlopen in python prior to 2.6.0 did not
|
||||
# have a timeout parameter
|
||||
r = urllib2.urlopen(request, None)
|
||||
else:
|
||||
r = urllib2.urlopen(request, None, timeout)
|
||||
|
||||
return r
|
||||
|
||||
#
|
||||
# Module-related functions
|
||||
#
|
||||
|
||||
def url_argument_spec():
|
||||
'''
|
||||
Creates an argument spec that can be used with any module
|
||||
that will be requesting content via urllib/urllib2
|
||||
'''
|
||||
return dict(
|
||||
url = dict(),
|
||||
force = dict(default='no', aliases=['thirsty'], type='bool'),
|
||||
http_agent = dict(default='ansible-httpget'),
|
||||
use_proxy = dict(default='yes', type='bool'),
|
||||
validate_certs = dict(default='yes', type='bool'),
|
||||
url_username = dict(required=False),
|
||||
url_password = dict(required=False),
|
||||
)
|
||||
|
||||
def fetch_url(module, url, data=None, headers=None, method=None,
|
||||
use_proxy=True, force=False, last_mod_time=None, timeout=10):
|
||||
'''
|
||||
Fetches a file from an HTTP/FTP server using urllib2. Requires the module environment
|
||||
'''
|
||||
|
||||
if not HAS_URLLIB2:
|
||||
module.fail_json(msg='urllib2 is not installed')
|
||||
elif not HAS_URLPARSE:
|
||||
module.fail_json(msg='urlparse is not installed')
|
||||
|
||||
# Get validate_certs from the module params
|
||||
validate_certs = module.params.get('validate_certs', True)
|
||||
|
||||
username = module.params.get('url_username', '')
|
||||
password = module.params.get('url_password', '')
|
||||
http_agent = module.params.get('http_agent', None)
|
||||
|
||||
r = None
|
||||
info = dict(url=url)
|
||||
try:
|
||||
if sys.version_info < (2,6,0):
|
||||
# urlopen in python prior to 2.6.0 did not
|
||||
# have a timeout parameter
|
||||
r = urllib2.urlopen(request, None)
|
||||
else:
|
||||
r = urllib2.urlopen(request, None, timeout)
|
||||
r = open_url(url, data=None, headers=None, method=None,
|
||||
use_proxy=True, force=False, last_mod_time=None, timeout=10,
|
||||
validate_certs=validate_certs, url_username=username,
|
||||
url_password=password, http_agent=http_agent)
|
||||
info.update(r.info())
|
||||
info['url'] = r.geturl() # The URL goes in too, because of redirects.
|
||||
info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
|
||||
except NoSSLError, e:
|
||||
distribution = get_distribution()
|
||||
if distribution.lower() == 'redhat':
|
||||
module.fail_json(msg='%s. You can also install python-ssl from EPEL' % str(e))
|
||||
except (ConnectionError, ValueError), e:
|
||||
module.fail_json(msg=str(e))
|
||||
except urllib2.HTTPError, e:
|
||||
info.update(dict(msg=str(e), status=e.code))
|
||||
except urllib2.URLError, e:
|
||||
|
|
@ -493,4 +528,3 @@ def fetch_url(module, url, data=None, headers=None, method=None,
|
|||
info.update(dict(msg="An unknown error occurred: %s" % str(e), status=-1))
|
||||
|
||||
return r, info
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue