ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

GCP Inventory Plugin (#36884)

Browse source 
* Inventory

* Adding multi project support

* Adding multi project support

* Fixing PR comments and cleaning code

* Adding cache support
* Changed filter notation

* Inventory changes

* Better readability for zones, networks, subnetworks
* Added project to inventory output
* Using IP instead of hostname
* Keyed_groups support
* Use all zones when none provided.

* PR changes

* Doc changes
* Accepts *gcp_compute.yaml file names
* Added support for changing host naming precedent

* PR changes round 2

* Cache changes
* Changed verify_file function
* Misc style changes

* Cache fixes

* Fix docs for `hostnames` option.
This commit is contained in:
Alex Stephen 2018-05-24 14:36:08 -05:00 committed by Ryan Brown
parent cba64f5869
commit dc31809d2a
2 changed files with 355 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/ansible/module_utils/gcp_utils.py
View file

@ -70,9 +70,10 @@ class GcpSession(object):
self.product = product
self._validate()
def get(self, url, body=None):
def get(self, url, body=None, **kwargs):
kwargs.update({'json': body, 'headers': self._headers()})
try:
return self.session().get(url, json=body, headers=self._headers())
return self.session().get(url, **kwargs)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
@ -105,12 +106,12 @@ class GcpSession(object):
if not HAS_GOOGLE_LIBRARIES:
self.module.fail_json(msg="Please install the google-auth library")
if self.module.params['service_account_email'] is not None and self.module.params['auth_kind'] != 'machineaccount':
if self.module.params.get('service_account_email') is not None and self.module.params['auth_kind'] != 'machineaccount':
self.module.fail_json(
msg="Service Acccount Email only works with Machine Account-based authentication"
)
if self.module.params['service_account_file'] is not None and self.module.params['auth_kind'] != 'serviceaccount':
if self.module.params.get('service_account_file') is not None and self.module.params['auth_kind'] != 'serviceaccount':
self.module.fail_json(
msg="Service Acccount File only works with Service Account-based authentication"
)

350
lib/ansible/plugins/inventory/gcp_compute.py Normal file
View file

@ -0,0 +1,350 @@
# Copyright (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
DOCUMENTATION = '''
name: gcp_compute
plugin_type: inventory
short_description: Google Cloud Compute Engine inventory source
requirements:
- requests >= 2.18.4
- google-auth >= 1.3.0
extends_documentation_fragment:
- gcp
- constructed
- inventory_cache
description:
- Get inventory hosts from Google Cloud Platform GCE.
- Uses a <name>.gcp.yaml (or <name>.gcp.yml) YAML configuration file.
options:
zones:
description: A list of regions in which to describe GCE instances.
default: all zones available to a given project
projects:
description: A list of projects in which to describe GCE instances.
filters:
description: >
A list of filter value pairs. Available filters are listed here
U(https://cloud.google.com/compute/docs/reference/rest/v1/instances/list).
Each additional filter in the list will act be added as an AND condition
(filter1 and filter2)
hostnames:
description: A list of options that describe the ordering for which
hostnames should be assigned. Currently supported hostnames are
'public_ip', 'private_ip', or 'name'.
default: ['public_ip', 'private_ip', 'name']
'''
EXAMPLES = '''
plugin: gcp_compute
zones: # populate inventory with instances in these regions
- us-east1-a
projects:
- gcp-prod-gke-100
- gcp-cicd-101
filters:
- machineType = n1-standard-1
- scheduling.automaticRestart = true AND machineType = n1-standard-1
scopes:
- https://www.googleapis.com/auth/compute
service_account_file: /tmp/service_account.json
auth_kind: serviceaccount
'''
from ansible.errors import AnsibleError, AnsibleParserError
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.six import string_types
from ansible.module_utils.gcp_utils import GcpSession, navigate_hash, GcpRequestException
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name
import json
# The mappings give an array of keys to get from the filter name to the value
# returned by boto3's GCE describe_instances method.
class GcpMockModule(object):
def __init__(self, params):
self.params = params
def fail_json(self, *args, **kwargs):
raise AnsibleError(kwargs['msg'])
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
NAME = 'gcp_compute'
def __init__(self):
super(InventoryModule, self).__init__()
self.group_prefix = 'gcp_'
def _populate_host(self, item):
'''
:param item: A GCP instance
'''
hostname = self._get_hostname(item)
self.inventory.add_host(hostname)
for key in item:
self.inventory.set_variable(hostname, key, item[key])
self.inventory.add_child('all', hostname)
def verify_file(self, path):
'''
:param path: the path to the inventory config file
:return the contents of the config file
'''
if super(InventoryModule, self).verify_file(path):
if path.endswith('.gcp.yml') or path.endswith('.gcp.yaml'):
return True
elif path.endswith('.gcp_compute.yml') or path.endswith('.gcp_compute.yaml'):
return True
return False
def self_link(self, params):
'''
:param params: a dict containing all of the fields relevant to build URL
:return the formatted URL as a string.
'''
return "https://www.googleapis.com/compute/v1/projects/{project}/zones/{zone}/instances".format(**params)
def fetch_list(self, params, link, query):
'''
:param params: a dict containing all of the fields relevant to build URL
:param link: a formatted URL
:param query: a formatted query string
:return the JSON response containing a list of instances.
'''
module = GcpMockModule(params)
auth = GcpSession(module, 'compute')
response = auth.get(link, params={'filter': query})
return self._return_if_object(module, response)
def _get_zones(self, config_data):
'''
:param config_data: dict of info from inventory file
:return an array of zones that this project has access to
'''
link = "https://www.googleapis.com/compute/v1/projects/{project}/zones".format(**config_data)
zones = []
zones_response = self.fetch_list(config_data, link, '')
for item in zones_response['items']:
zones.append(item['name'])
return zones
def _get_query_options(self, filters):
'''
:param config_data: contents of the inventory config file
:return A fully built query string
'''
if not filters:
return ''
if len(filters) == 1:
return filters[0]
else:
queries = []
for f in filters:
# For multiple queries, all queries should have ()
if f[0] != '(' and f[-1] != ')':
queries.append("(%s)" % ''.join(f))
else:
queries.append(f)
return ' '.join(queries)
def _return_if_object(self, module, response):
'''
:param module: A GcpModule
:param response: A Requests response object
:return JSON response
'''
# If not found, return nothing.
if response.status_code == 404:
return None
# If no content, return nothing.
if response.status_code == 204:
return None
try:
response.raise_for_status
result = response.json()
except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst:
module.fail_json(msg="Invalid JSON response with error: %s" % inst)
except GcpRequestException as inst:
module.fail_json(msg="Network error: %s" % inst)
if navigate_hash(result, ['error', 'errors']):
module.fail_json(msg=navigate_hash(result, ['error', 'errors']))
if result['kind'] != 'compute#instanceList' and result['kind'] != 'compute#zoneList':
module.fail_json(msg="Incorrect result: {kind}".format(**result))
return result
def _format_items(self, items):
'''
:param items: A list of hosts
'''
for host in items:
if 'zone' in host:
host['zone_selflink'] = host['zone']
host['zone'] = host['zone'].split('/')[-1]
if 'machineType' in host:
host['machineType_selflink'] = host['machineType']
host['machineType'] = host['machineType'].split('/')[-1]
if 'networkInterfaces' in host:
for network in host['networkInterfaces']:
if 'network' in network:
network['network'] = self._format_network_info(network['network'])
if 'subnetwork' in network:
network['subnetwork'] = self._format_network_info(network['subnetwork'])
host['project'] = host['selfLink'].split('/')[6]
return items
def _add_hosts(self, items, config_data, format_items=True):
'''
:param items: A list of hosts
:param config_data: configuration data
:param format_items: format items or not
'''
if not items:
return
if format_items:
items = self._format_items(items)
for host in items:
self._populate_host(host)
hostname = self._get_hostname(host)
self._set_composite_vars(self.get_option('compose'), host, hostname)
self._add_host_to_composed_groups(self.get_option('groups'), host, hostname)
self._add_host_to_keyed_groups(self.get_option('keyed_groups'), host, hostname)
def _format_network_info(self, address):
'''
:param address: A GCP network address
:return a dict with network shortname and region
'''
split = address.split('/')
region = ''
if 'global' in split:
region = 'global'
else:
region = split[8]
return {
'region': region,
'name': split[-1],
'selfLink': address
}
def _get_hostname(self, item):
'''
:param item: A host response from GCP
:return the hostname of this instance
'''
hostname_ordering = ['public_ip', 'private_ip', 'name']
if self.get_option('hostnames'):
hostname_ordering = self.get_option('hostnames')
for order in hostname_ordering:
name = None
if order == 'public_ip':
name = self._get_publicip(item)
elif order == 'private_ip':
name = self._get_privateip(item)
elif order == 'name':
name = item[u'name']
else:
raise AnsibleParserError("%s is not a valid hostname precedent" % order)
if name:
return name
raise AnsibleParserError("No valid name found for host")
def _get_publicip(self, item):
'''
:param item: A host response from GCP
:return the publicIP of this instance or None
'''
# Get public IP if exists
for interface in item['networkInterfaces']:
if 'accessConfigs' in interface:
for accessConfig in interface['accessConfigs']:
if 'natIP' in accessConfig:
return accessConfig[u'natIP']
return None
def _get_privateip(self, item):
'''
:param item: A host response from GCP
:return the privateIP of this instance or None
'''
# Fallback: Get private IP
for interface in item[u'networkInterfaces']:
if 'networkIP' in interface:
return interface[u'networkIP']
def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path)
config_data = {}
if self.verify_file(path):
config_data = self._read_config_data(path)
# get user specifications
if 'zones' in config_data:
if not isinstance(config_data['zones'], list):
raise AnsibleParserError("Zones must be a list in GCP inventory YAML files")
# get user specifications
if 'projects' not in config_data:
raise AnsibleParserError("Projects must be included in inventory YAML file")
if not isinstance(config_data['projects'], list):
raise AnsibleParserError("Projects must be a list in GCP inventory YAML files")
projects = config_data['projects']
zones = config_data.get('zones')
config_data['scopes'] = ['https://www.googleapis.com/auth/compute']
query = self._get_query_options(config_data['filters'])
# Cache logic
if cache:
cache = self.get_option('cache')
cache_key = self.get_cache_key(path)
else:
cache_key = None
cache_needs_update = False
if cache:
try:
results = self.cache.get(cache_key)
for project in results:
for zone in results[project]:
self._add_hosts(results[project][zone], config_data, False)
except KeyError:
cache_needs_update = True
if not cache or cache_needs_update:
cached_data = {}
for project in projects:
cached_data[project] = {}
config_data['project'] = project
if not zones:
zones = self._get_zones(config_data)
for zone in zones:
config_data['zone'] = zone
link = self.self_link(config_data)
resp = self.fetch_list(config_data, link, query)
self._add_hosts(resp.get('items'), config_data)
cached_data[project][zone] = resp.get('items')
if cache_needs_update:
self.cache.set(cache_key, cached_data)