ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

Adding auth support for google-api-python-client and gcloud-python (#19090)

Browse source 
Support for the Google API and GCloud-Python Clients have been added.

The three libraries:
* GCloud-Python: A new function, get_google_cloud_credentials, should be used.  The credentials-object returned can be passed to any gcloud-python client.  Using this client library requires in the installation of gcloud-python.  This is preferred library for new modules.

* Google API: A new function, gcp_api_auth, should be used to take advantage of services requiring this client.  This client library should be used if the desired functionality is not available in GCloud-Python.  Using this library requires the installation of google-api-python-client.

* libcloud: Existing function, gcp_connect, should be used.  The interface and return values have not changed and existing modules (such as gce, gce_pd and gce_net) should work without modification.  Note that the credentials-fetching code has been refactored out of gcp_connect so that can be reused by all connection functions. To use this function, apache-libcloud must be installed.

Import guards have been added and will only be trigger if a user tries to use a function that is missing dependencies.

Credential-specifying mechanisms (i.e, ansible module params, env vars and libcloud secrets.py) have not changed.  They have been refactored and unit tests have been added to allow for changes going forward. We are deprecating (and removing in a subsequent release) the ability to specify credentials via the libcloud secrets file.  Also, we have deprecated (and also plan to remove in a subsequent release) the ability to use a p12 pem file for a key - the JSON format is strongly preferred.  Deprecation warnings have been added for both of these issues (see the Ansible docs on how to disable deprecation warnings).
This commit is contained in:
Tom Melendez 2016-12-29 09:33:52 -08:00 committed by Ryan Brown
parent 65f0155d8a
commit 7e88df7ebc
2 changed files with 473 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

323
lib/ansible/module_utils/gcp.py
View file

@ -32,41 +32,97 @@ import os
import traceback
from distutils.version import LooseVersion
# libcloud
try:
import libcloud
HAS_LIBCLOUD_BASE = True
except ImportError:
HAS_LIBCLOUD_BASE = False
def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_version):
"""Return a Google Cloud Platform connection."""
if not HAS_LIBCLOUD_BASE:
module.fail_json(msg='libcloud must be installed to use this module')
# google-python-api
try:
from httplib2 import Http
from oauth2client.service_account import ServiceAccountCredentials
from googleapiclient.http import set_user_agent
HAS_GOOGLE_API_LIB = True
except ImportError:
HAS_GOOGLE_API_LIB = False
# google-cloud-python
try:
import google.cloud.core as _GOOGLE_CLOUD_CORE_CHECK__
from httplib2 import Http
HAS_GOOGLE_CLOUD_CORE = True
except ImportError:
HAS_GOOGLE_CLOUD_CORE = False
# Ansible Display object for warnings
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
def _get_gcp_ansible_credentials(module):
"""Helper to fetch creds from AnsibleModule object."""
service_account_email = module.params.get('service_account_email', None)
credentials_file = module.params.get('credentials_file', None)
pem_file = module.params.get('pem_file', None)
# Note: pem_file is discouraged and will be deprecated
credentials_file = module.params.get('pem_file', None) or module.params.get(
'credentials_file', None)
project_id = module.params.get('project_id', None)
return (service_account_email, credentials_file, project_id)
def _get_gcp_environ_var(var_name, default_value):
"""Wrapper around os.environ.get call."""
return os.environ.get(
var_name, default_value)
def _get_gcp_environment_credentials(service_account_email, credentials_file, project_id):
"""Helper to look in environment variables for credentials."""
# If any of the values are not given as parameters, check the appropriate
# environment variables.
if not service_account_email:
service_account_email = os.environ.get('GCE_EMAIL', None)
if not project_id:
project_id = os.environ.get('GCE_PROJECT', None)
if not pem_file:
pem_file = os.environ.get('GCE_PEM_FILE_PATH', None)
service_account_email = _get_gcp_environ_var('GCE_EMAIL', None)
if not credentials_file:
credentials_file = os.environ.get('GCE_CREDENTIALS_FILE_PATH', pem_file)
credentials_file = _get_gcp_environ_var(
'GCE_CREDENTIALS_FILE_PATH', None) or _get_gcp_environ_var(
'GOOGLE_APPLICATION_CREDENTIALS', None) or _get_gcp_environ_var(
'GCE_PEM_FILE_PATH', None)
if not project_id:
project_id = _get_gcp_environ_var('GCE_PROJECT', None) or _get_gcp_environ_var(
'GOOGLE_CLOUD_PROJECT', None)
return (service_account_email, credentials_file, project_id)
# If we still don't have one or more of our credentials, attempt to
# get the remaining values from the libcloud secrets file.
if service_account_email is None or pem_file is None:
def _get_gcp_libcloud_credentials(service_account_email=None, credentials_file=None, project_id=None):
"""
Helper to look for libcloud secrets.py file.
Note: This has an 'additive' effect right now, filling in
vars not specified elsewhere, in order to keep legacy functionality.
This method of specifying credentials will be deprecated, otherwise
we'd look to make it more restrictive with an all-vars-or-nothing approach.
:param service_account: GCP service account email used to make requests
:type service_account: ``str`` or None
:param credentials_file: Path on disk to credentials file
:type credentials_file: ``str`` or None
:param project_id: GCP project ID.
:type project_id: ``str`` or None
:return: tuple of (service_account, credentials_file, project_id)
:rtype: ``tuple`` of ``str``
"""
if service_account_email is None or credentials_file is None:
try:
import secrets
display.deprecated(msg=("secrets file found at '%s'. This method of specifying "
"credentials is deprecated. Please use env vars or "
"Ansible YAML files instead" % (secrets.__file__)), version=2.5)
except ImportError:
secrets = None
if hasattr(secrets, 'GCE_PARAMS'):
if not service_account_email:
service_account_email = secrets.GCE_PARAMS[0]
@ -75,34 +131,139 @@ def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_ver
keyword_params = getattr(secrets, 'GCE_KEYWORD_PARAMS', {})
if not project_id:
project_id = keyword_params.get('project', None)
return (service_account_email, credentials_file, project_id)
# If we *still* don't have the credentials we need, then it's time to
# just fail out.
if service_account_email is None or credentials_file is None or project_id is None:
module.fail_json(msg='Missing GCE connection parameters in libcloud '
def _get_gcp_credentials(module, require_valid_json=True, check_libcloud=False):
"""
Obtain GCP credentials by trying various methods.
There are 3 ways to specify GCP credentials:
1. Specify via Ansible module parameters (recommended).
2. Specify via environment variables. Two sets of env vars are available:
a) GOOGLE_CLOUD_PROJECT, GOOGLE_CREDENTIALS_APPLICATION (preferred)
b) GCE_PROJECT, GCE_CREDENTIAL_FILE_PATH, GCE_EMAIL (legacy, not recommended; req'd if
using p12 key)
3. Specify via libcloud secrets.py file (deprecated).
There are 3 helper functions to assist in the above.
Regardless of method, the user also has the option of specifying a JSON
file or a p12 file as the credentials file. JSON is strongly recommended and
p12 will be removed in the future.
Additionally, flags may be set to require valid json and check the libcloud
version.
:param module: initialized Ansible module object
:type module: `class AnsibleModule`
:param require_valid_json: If true, require credentials to be valid JSON. Default is True.
:type require_valid_json: ``bool``
:params check_libcloud: If true, check the libcloud version available to see if
JSON creds are supported.
:type check_libcloud: ``bool``
:return: {'service_account_email': service_account_email,
'credentials_file': credentials_file,
'project_id': project_id}
:rtype: ``dict``
"""
(service_account_email,
credentials_file,
project_id) = _get_gcp_ansible_credentials(module)
# If any of the values are not given as parameters, check the appropriate
# environment variables.
(service_account_email,
credentials_file,
project_id) = _get_gcp_environment_credentials(service_account_email,
credentials_file, project_id)
# If we still don't have one or more of our credentials, attempt to
# get the remaining values from the libcloud secrets file.
(service_account_email,
credentials_file,
project_id) = _get_gcp_libcloud_credentials(service_account_email,
credentials_file, project_id)
if credentials_file is None or project_id is None or service_account_email is None:
if check_libcloud is True:
# TODO(supertom): this message is legacy and integration tests depend on it.
module.fail_json(msg='Missing GCE connection parameters in libcloud '
'secrets file.')
return None
else:
# We have credentials but lets make sure that if they are JSON we have the minimum
# libcloud requirement met
try:
# Try to read credentials as JSON
with open(credentials_file) as credentials:
json.loads(credentials.read())
return None
else:
if credentials_file is None or project_id is None:
module.fail_json(msg=('GCP connection error: enable to determine project (%s) or'
'credentials file (%s)' % (project_id, credentials_file)))
# ensure the credentials file is found and is in the proper format.
_validate_credentials_file(module, credentials_file,
require_valid_json=require_valid_json,
check_libcloud=check_libcloud)
return {'service_account_email': service_account_email,
'credentials_file': credentials_file,
'project_id': project_id}
def _validate_credentials_file(module, credentials_file, require_valid_json=True, check_libcloud=False):
"""
Check for valid credentials file.
Optionally check for JSON format and if libcloud supports JSON.
:param module: initialized Ansible module object
:type module: `class AnsibleModule`
:param credentials_file: path to file on disk
:type credentials_file: ``str``. Complete path to file on disk.
:param require_valid_json: If true, require credentials to be valid JSON. Default is True.
:type require_valid_json: ``bool``
:params check_libcloud: If true, check the libcloud version available to see if
JSON creds are supported.
:type check_libcloud: ``bool``
:returns: True
:rtype: ``bool``
"""
try:
# Try to read credentials as JSON
with open(credentials_file) as credentials:
json.loads(credentials.read())
# If the credentials are proper JSON and we do not have the minimum
# required libcloud version, bail out and return a descriptive error
if LooseVersion(libcloud.__version__) < '0.17.0':
if check_libcloud and LooseVersion(libcloud.__version__) < '0.17.0':
module.fail_json(msg='Using JSON credentials but libcloud minimum version not met. '
'Upgrade to libcloud>=0.17.0.')
return None
except ValueError as e:
# Not JSON
pass
return True
except IOError as e:
module.fail_json(msg='GCP Credentials File %s not found.', changed=False)
return False
except ValueError as e:
if require_valid_json:
module.fail_json(msg='GCP Credentials File %s invalid. Must be valid JSON.', changed=False)
else:
display.deprecated(msg=("Non-JSON credentials file provided. This format is deprecated. "
" Please generate a new JSON key from the Google Cloud console"),
version=2.5)
return True
def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_version):
"""Return a Google libcloud driver connection."""
if not HAS_LIBCLOUD_BASE:
module.fail_json(msg='libcloud must be installed to use this module')
creds = _get_gcp_credentials(module,
require_valid_json=False,
check_libcloud=True)
try:
gcp = get_driver(provider)(service_account_email, credentials_file,
gcp = get_driver(provider)(creds['service_account_email'], creds['credentials_file'],
datacenter=module.params.get('zone', None),
project=project_id)
project=creds['project_id'])
gcp.connection.user_agent_append("%s/%s" % (
user_agent_product, user_agent_version))
except (RuntimeError, ValueError) as e:
@ -112,6 +273,98 @@ def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_ver
return gcp
def get_google_cloud_credentials(module, scopes=[]):
"""
Get credentials object for use with Google Cloud client.
To connect via libcloud, don't use this function, use gcp_connect instead. For
Google Python API Client, see get_google_api_auth for how to connect.
For more information on Google's client library options for Python, see:
U(https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries)
Google Cloud example:
creds, params = get_google_cloud_credentials(module, scopes, user_agent_product, user_agent_version)
pubsub_client = pubsub.Client(project=params['project_id'], credentials=creds)
pubsub_client.user_agent = 'ansible-pubsub-0.1'
...
:param module: initialized Ansible module object
:type module: `class AnsibleModule`
:param scopes: list of scopes
:type module: ``list`` of URIs
:returns: A tuple containing (google authorized) credentials object and
params dict {'service_account_email': '...', 'credentials_file': '...', 'project_id': ...}
:rtype: ``tuple``
"""
creds = _get_gcp_credentials(module,
require_valid_json=True,
check_libcloud=False)
try:
credentials = ServiceAccountCredentials.from_json_keyfile_name(
creds['credentials_file'],
scopes=scopes
)
return (credentials, creds)
except Exception as e:
module.fail_json(msg=unexpected_error_msg(e), changed=False)
return (None, None)
def get_google_api_auth(module, scopes=[], user_agent_product='ansible-python-api', user_agent_version='NA'):
"""
Authentication for use with google-python-api-client.
Function calls _get_gcp_credentials, which attempts to assemble the credentials from various locations.
Next it attempts to authenticate with Google.
This function returns an httplib2 object that can be provided to the Google Python API client.
For libcloud, don't use this function, use gcp_connect instead. For Google Cloud, See
get_google_cloud_credentials for how to connect.
For more information on Google's client library options for Python, see:
U(https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries)
Google API example:
http_auth, conn_params = gcp_api_auth(module, scopes, user_agent_product, user_agent_version)
service = build('myservice', 'v1', http=http_auth)
...
:param module: initialized Ansible module object
:type module: `class AnsibleModule`
:param scopes: list of scopes
:type scopes: ``list`` of URIs
:param user_agent_product: User agent product. eg: 'ansible-python-api'
:type user_agent_product: ``str``
:param user_agent_version: Version string to append to product. eg: 'NA' or '0.1'
:type user_agent_version: ``str``
:returns: A tuple containing (google authorized) httplib2 request object and a
params dict {'service_account_email': '...', 'credentials_file': '...', 'project_id': ...}
:rtype: ``tuple``
"""
if not HAS_GOOGLE_API_LIB:
module.fail_json(msg="Please install google-api-python-client library")
# TODO(supertom): verify scopes
if not scopes:
scopes = ['https://www.googleapis.com/auth/cloud-platform']
try:
(credentials, conn_params) = get_google_credentials(module, scopes,
require_valid_json=True, check_libcloud=False)
http = set_user_agent(Http(), '%s-%s' % (user_agent_product, user_agent_version))
http_auth = credentials.authorize(http)
return (http_auth, conn_params)
except Exception as e:
module.fail_json(msg=unexpected_error_msg(e), changed=False)
return (None, None)
def unexpected_error_msg(error):
"""Create an error string based on passed in error."""
return 'Unexpected response: (%s). Detail: %s' % (str(error), traceback.format_exc(error))

185
test/units/module_utils/gcp/test_auth.py Normal file
View file

@ -0,0 +1,185 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# (c) 2016, Tom Melendez <tom@supertom.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
from ansible.compat.tests import mock, unittest
from ansible.module_utils.gcp import (_get_gcp_ansible_credentials, _get_gcp_environ_var,
_get_gcp_libcloud_credentials, _get_gcp_environment_credentials,
_validate_credentials_file)
# Fake data/function used for testing
fake_env_data = {'GCE_EMAIL': 'gce-email'}
def fake_get_gcp_environ_var(var_name, default_value):
if var_name not in fake_env_data:
return default_value
else:
return fake_env_data[var_name]
class GCPAuthTestCase(unittest.TestCase):
"""Tests to verify different Auth mechanisms."""
def test_get_gcp_ansible_credentials(self):
# create a fake (AnsibleModule) object to pass to the function
class FakeModule(object):
class Params():
data = {}
def get(self, key, alt=None):
if key in self.data:
return self.data[key]
else:
return alt
def __init__(self, data={}):
self.params = FakeModule.Params()
self.params.data = data
input_data = {'service_account_email': 'mysa',
'credentials_file': 'path-to-file.json',
'project_id': 'my-cool-project'}
module = FakeModule(input_data)
actual = _get_gcp_ansible_credentials(module)
expected = tuple(input_data.values())
self.assertEqual(sorted(expected), sorted(actual))
def test_get_gcp_environ_var(self):
# Chose not to mock this so we could really verify that it
# works as expected.
existing_var_name = 'gcp_ansible_auth_test_54321'
non_existing_var_name = 'doesnt_exist_gcp_ansible_auth_test_12345'
os.environ[existing_var_name] = 'foobar'
self.assertEqual('foobar', _get_gcp_environ_var(existing_var_name, None))
del os.environ[existing_var_name]
self.assertEqual('default_value', _get_gcp_environ_var(
non_existing_var_name, 'default_value'))
def test_get_gcp_libcloud_credentials_no_import(self):
"""No secrets imported. Whatever is sent in should come out."""
actual = _get_gcp_libcloud_credentials(service_account_email=None,
credentials_file=None,
project_id=None)
expected = (None, None, None)
self.assertEqual(expected, actual)
# no libcloud, with values
actual = _get_gcp_libcloud_credentials(service_account_email='sa-email',
credentials_file='creds-file',
project_id='proj-id')
expected = ('sa-email', 'creds-file', 'proj-id')
self.assertEqual(expected, actual)
@mock.patch("ansible.utils.display.Display.deprecated",
name='mock_deprecated', return_value=None)
def test_get_gcp_libcloud_credentials_import(self, mock_deprecated):
"""secrets is imported and those values should be used."""
# Note: Opted for a real class here rather than MagicMock as
# __getitem__ comes for free.
class FakeSecrets:
def __init__(self):
# 2 element list, service account email and creds file
self.GCE_PARAMS = ['secrets-sa', 'secrets-file.json']
# dictionary with project_id, optionally auth_type
self.GCE_KEYWORD_PARAMS = {}
self.__file__ = 'THIS_IS_A_FAKEFILE_FOR_TESTING'
# patch in module
fake_secrets = FakeSecrets()
patcher = mock.patch.dict(sys.modules,{'secrets': fake_secrets})
patcher.start()
# obtain sa and creds from secrets
actual = _get_gcp_libcloud_credentials(service_account_email=None,
credentials_file=None,
project_id='proj-id')
expected = ('secrets-sa', 'secrets-file.json', 'proj-id')
self.assertEqual(expected, actual)
# fetch project id. Current logic requires sa-email or creds to be set.
fake_secrets.GCE_KEYWORD_PARAMS['project'] = 'new-proj-id'
fake_secrets.GCE_PARAMS[1] = 'my-creds.json'
actual = _get_gcp_libcloud_credentials(service_account_email='my-sa',
credentials_file=None,
project_id=None)
expected = ('my-sa', 'my-creds.json', 'new-proj-id')
self.assertEqual(expected, actual)
# stop patching
patcher.stop()
@mock.patch("ansible.utils.display.Display.deprecated",
name='mock_deprecated', return_value=None)
def test_validate_credentials_file(self, mock_deprecated):
# TODO(supertom): Only dealing with p12 here, check the other states
# of this function
module = mock.MagicMock()
with mock.patch("ansible.module_utils.gcp.open",
mock.mock_open(read_data='foobar'), create=True) as m:
# pem condition, warning is surpressed with the return_value
credentials_file = '/foopath/pem.pem'
is_valid = _validate_credentials_file(module,
credentials_file=credentials_file,
require_valid_json=False,
check_libcloud=False)
self.assertTrue(is_valid)
@mock.patch('ansible.module_utils.gcp._get_gcp_environ_var',
side_effect=fake_get_gcp_environ_var)
def test_get_gcp_environment_credentials(self, mockobj):
global fake_env_data
actual = _get_gcp_environment_credentials(None, None, None)
expected = tuple(['gce-email', None, None])
self.assertEqual(expected, actual)
fake_env_data = {'GCE_PEM_FILE_PATH': '/path/to/pem.pem'}
expected = tuple([None, '/path/to/pem.pem', None])
actual = _get_gcp_environment_credentials(None, None, None)
self.assertEqual(expected, actual)
# pem and creds are set, expect creds
fake_env_data = {'GCE_PEM_FILE_PATH': '/path/to/pem.pem',
'GCE_CREDENTIALS_FILE_PATH': '/path/to/creds.json'}
expected = tuple([None, '/path/to/creds.json', None])
actual = _get_gcp_environment_credentials(None, None, None)
self.assertEqual(expected, actual)
# expect GOOGLE_APPLICATION_CREDENTIALS over PEM
fake_env_data = {'GCE_PEM_FILE_PATH': '/path/to/pem.pem',
'GOOGLE_APPLICATION_CREDENTIALS': '/path/to/appcreds.json'}
expected = tuple([None, '/path/to/appcreds.json', None])
actual = _get_gcp_environment_credentials(None, None, None)
self.assertEqual(expected, actual)
# project tests
fake_env_data = {'GCE_PROJECT': 'my-project'}
expected = tuple([None, None, 'my-project'])
actual = _get_gcp_environment_credentials(None, None, None)
self.assertEqual(expected, actual)
fake_env_data = {'GOOGLE_CLOUD_PROJECT': 'my-cloud-project'}
expected = tuple([None, None, 'my-cloud-project'])
actual = _get_gcp_environment_credentials(None, None, None)
self.assertEqual(expected, actual)
# data passed in, picking up project id only
fake_env_data = {'GOOGLE_CLOUD_PROJECT': 'my-project'}
expected = tuple(['my-sa-email', '/path/to/creds.json', 'my-project'])
actual = _get_gcp_environment_credentials('my-sa-email', '/path/to/creds.json', None)
self.assertEqual(expected, actual)
if __name__ == '__main__':
unittest.main()