community.general/scripts/inventory/collins.py

#!/usr/bin/env python

"""
Collins external inventory script
=================================

Ansible has a feature where instead of reading from /etc/ansible/hosts
as a text file, it can query external programs to obtain the list
of hosts, groups the hosts are in, and even variables to assign to each host.

Collins is a hardware asset management system originally developed by
Tumblr for tracking new hardware as it built out its own datacenters.  It
exposes a rich API for manipulating and querying one's hardware inventory,
which makes it an ideal 'single point of truth' for driving systems
automation like Ansible.  Extensive documentation on Collins, including a quickstart,
API docs, and a full reference manual, can be found here:

http://tumblr.github.io/collins

This script adds support to Ansible for obtaining a dynamic inventory of
assets in your infrastructure, grouping them in Ansible by their useful attributes,
and binding all facts provided by Collins to each host so that they can be used to
drive automation.  Some parts of this script were cribbed shamelessly from mdehaan's
Cobbler inventory script.

To use it, copy it to your repo and pass -i <collins script> to the ansible or
ansible-playbook command; if you'd like to use it by default, simply copy collins.ini
to /etc/ansible and this script to /etc/ansible/hosts.

Alongside the options set in collins.ini, there are several environment variables
that will be used instead of the configured values if they are set:

 - COLLINS_USERNAME - specifies a username to use for Collins authentication
 - COLLINS_PASSWORD - specifies a password to use for Collins authentication
 - COLLINS_ASSET_TYPE - specifies a Collins asset type to use during querying;
   this can be used to run Ansible automation against different asset classes than
   server nodes, such as network switches and PDUs
 - COLLINS_CONFIG - specifies an alternative location for collins.ini, defaults to
   <location of collins.py>/collins.ini

If errors are encountered during operation, this script will return an exit code of
255; otherwise, it will return an exit code of 0.

Collins attributes are accessible as variables in ansible via the COLLINS['attribute_name'].

Tested against Ansible 1.8.2 and Collins 1.3.0.
"""

# (c) 2014, Steve Salevan <steve.salevan@gmail.com>
#
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

######################################################################


import argparse
import logging
import os
import re
import sys
from time import time
import traceback

import json

from ansible.module_utils.six import iteritems
from ansible.module_utils.six.moves import configparser as ConfigParser
from ansible.module_utils.six.moves.urllib.parse import urlencode

from ansible.module_utils.urls import open_url


class CollinsDefaults(object):
    ASSETS_API_ENDPOINT = '%s/api/assets'
    SPECIAL_ATTRIBUTES = set([
        'CREATED',
        'DELETED',
        'UPDATED',
        'STATE',
    ])
    LOG_FORMAT = '%(asctime)-15s %(message)s'


class Error(Exception):
    pass


class MaxRetriesError(Error):
    pass


class CollinsInventory(object):

    def __init__(self):
        """ Constructs CollinsInventory object and reads all configuration. """

        self.inventory = dict()  # A list of groups and the hosts in that group
        self.cache = dict()  # Details about hosts in the inventory

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        logging.basicConfig(format=CollinsDefaults.LOG_FORMAT,
                            filename=self.log_location)
        self.log = logging.getLogger('CollinsInventory')

    def _asset_get_attribute(self, asset, attrib):
        """ Returns a user-defined attribute from an asset if it exists; otherwise,
            returns None. """

        if 'ATTRIBS' in asset:
            for attrib_block in asset['ATTRIBS'].keys():
                if attrib in asset['ATTRIBS'][attrib_block]:
                    return asset['ATTRIBS'][attrib_block][attrib]
        return None

    def _asset_has_attribute(self, asset, attrib):
        """ Returns whether a user-defined attribute is present on an asset. """

        if 'ATTRIBS' in asset:
            for attrib_block in asset['ATTRIBS'].keys():
                if attrib in asset['ATTRIBS'][attrib_block]:
                    return True
        return False

    def run(self):
        """ Main execution path """

        # Updates cache if cache is not present or has expired.
        successful = True
        if self.args.refresh_cache:
            successful = self.update_cache()
        elif not self.is_cache_valid():
            successful = self.update_cache()
        else:
            successful = self.load_inventory_from_cache()
            successful &= self.load_cache_from_cache()

        data_to_print = ""

        # Data to print
        if self.args.host:
            data_to_print = self.get_host_info()

        elif self.args.list:
            # Display list of instances for inventory
            data_to_print = self.json_format_dict(self.inventory, self.args.pretty)

        else:  # default action with no options
            data_to_print = self.json_format_dict(self.inventory, self.args.pretty)

        print(data_to_print)
        return successful

    def find_assets(self, attributes=None, operation='AND'):
        """ Obtains Collins assets matching the provided attributes. """
        attributes = {} if attributes is None else attributes

        # Formats asset search query to locate assets matching attributes, using
        # the CQL search feature as described here:
        # http://tumblr.github.io/collins/recipes.html
        attributes_query = ['='.join(attr_pair) for attr_pair in iteritems(attributes)]
        query_parameters = {
            'details': ['True'],
            'operation': [operation],
            'query': attributes_query,
            'remoteLookup': [str(self.query_remote_dcs)],
            'size': [self.results_per_query],
            'type': [self.collins_asset_type],
        }
        assets = []
        cur_page = 0
        num_retries = 0
        # Locates all assets matching the provided query, exhausting pagination.
        while True:
            if num_retries == self.collins_max_retries:
                raise MaxRetriesError("Maximum of %s retries reached; giving up" % self.collins_max_retries)
            query_parameters['page'] = cur_page
            query_url = "%s?%s" % (
                (CollinsDefaults.ASSETS_API_ENDPOINT % self.collins_host),
                urlencode(query_parameters, doseq=True)
            )
            try:
                response = open_url(query_url,
                                    timeout=self.collins_timeout_secs,
                                    url_username=self.collins_username,
                                    url_password=self.collins_password,
                                    force_basic_auth=True)
                json_response = json.loads(response.read())
                # Adds any assets found to the array of assets.
                assets += json_response['data']['Data']
                # If we've retrieved all of our assets, breaks out of the loop.
                if len(json_response['data']['Data']) == 0:
                    break
                cur_page += 1
                num_retries = 0
            except Exception:
                self.log.error("Error while communicating with Collins, retrying:\n%s", traceback.format_exc())
                num_retries += 1
        return assets

    def is_cache_valid(self):
        """ Determines if the cache files have expired, or if it is still valid """

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_inventory):
                    return True

        return False

    def read_settings(self):
        """ Reads the settings from the collins.ini file """

        config_loc = os.getenv('COLLINS_CONFIG', os.path.dirname(os.path.realpath(__file__)) + '/collins.ini')

        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/collins.ini')

        self.collins_host = config.get('collins', 'host')
        self.collins_username = os.getenv('COLLINS_USERNAME', config.get('collins', 'username'))
        self.collins_password = os.getenv('COLLINS_PASSWORD', config.get('collins', 'password'))
        self.collins_asset_type = os.getenv('COLLINS_ASSET_TYPE', config.get('collins', 'asset_type'))
        self.collins_timeout_secs = config.getint('collins', 'timeout_secs')
        self.collins_max_retries = config.getint('collins', 'max_retries')

        self.results_per_query = config.getint('collins', 'results_per_query')
        self.ip_address_index = config.getint('collins', 'ip_address_index')
        self.query_remote_dcs = config.getboolean('collins', 'query_remote_dcs')
        self.prefer_hostnames = config.getboolean('collins', 'prefer_hostnames')

        cache_path = config.get('collins', 'cache_path')
        self.cache_path_cache = cache_path + \
            '/ansible-collins-%s.cache' % self.collins_asset_type
        self.cache_path_inventory = cache_path + \
            '/ansible-collins-%s.index' % self.collins_asset_type
        self.cache_max_age = config.getint('collins', 'cache_max_age')

        log_path = config.get('collins', 'log_path')
        self.log_location = log_path + '/ansible-collins.log'

    def parse_cli_args(self):
        """ Command line argument processing """

        parser = argparse.ArgumentParser(
            description='Produces an Ansible Inventory file based on Collins')
        parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
        parser.add_argument('--host', action='store', help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to Collins '
                                 '(default: False - use cache files)')
        parser.add_argument('--pretty', action='store_true', default=False, help='Pretty print all JSON output')
        self.args = parser.parse_args()

    def update_cache(self):
        """ Make calls to Collins and saves the output in a cache """

        self.cache = dict()
        self.inventory = dict()

        # Locates all server assets from Collins.
        try:
            server_assets = self.find_assets()
        except Exception:
            self.log.error("Error while locating assets from Collins:\n%s", traceback.format_exc())
            return False

        for asset in server_assets:
            # Determines the index to retrieve the asset's IP address either by an
            # attribute set on the Collins asset or the pre-configured value.
            if self._asset_has_attribute(asset, 'ANSIBLE_IP_INDEX'):
                ip_index = self._asset_get_attribute(asset, 'ANSIBLE_IP_INDEX')
                try:
                    ip_index = int(ip_index)
                except Exception:
                    self.log.error(
                        "ANSIBLE_IP_INDEX attribute on asset %s not an integer: %s", asset,
                        ip_index)
            else:
                ip_index = self.ip_address_index

            asset['COLLINS'] = {}

            # Attempts to locate the asset's primary identifier (hostname or IP address),
            # which will be used to index the asset throughout the Ansible inventory.
            if self.prefer_hostnames and self._asset_has_attribute(asset, 'HOSTNAME'):
                asset_identifier = self._asset_get_attribute(asset, 'HOSTNAME')
            elif 'ADDRESSES' not in asset:
                self.log.warning("No IP addresses found for asset '%s', skipping", asset)
                continue
            elif len(asset['ADDRESSES']) < ip_index + 1:
                self.log.warning(
                    "No IP address found at index %s for asset '%s', skipping",
                    ip_index, asset)
                continue
            else:
                asset_identifier = asset['ADDRESSES'][ip_index]['ADDRESS']

            # Adds an asset index to the Ansible inventory based upon unpacking
            # the name of the asset's current STATE from its dictionary.
            if 'STATE' in asset['ASSET'] and asset['ASSET']['STATE']:
                state_inventory_key = self.to_safe(
                    'STATE-%s' % asset['ASSET']['STATE']['NAME'])
                self.push(self.inventory, state_inventory_key, asset_identifier)

            # Indexes asset by all user-defined Collins attributes.
            if 'ATTRIBS' in asset:
                for attrib_block in asset['ATTRIBS'].keys():
                    for attrib in asset['ATTRIBS'][attrib_block].keys():
                        asset['COLLINS'][attrib] = asset['ATTRIBS'][attrib_block][attrib]
                        attrib_key = self.to_safe('%s-%s' % (attrib, asset['ATTRIBS'][attrib_block][attrib]))
                        self.push(self.inventory, attrib_key, asset_identifier)

            # Indexes asset by all built-in Collins attributes.
            for attribute in asset['ASSET'].keys():
                if attribute not in CollinsDefaults.SPECIAL_ATTRIBUTES:
                    attribute_val = asset['ASSET'][attribute]
                    if attribute_val is not None:
                        attrib_key = self.to_safe('%s-%s' % (attribute, attribute_val))
                        self.push(self.inventory, attrib_key, asset_identifier)

            # Indexes asset by hardware product information.
            if 'HARDWARE' in asset:
                if 'PRODUCT' in asset['HARDWARE']['BASE']:
                    product = asset['HARDWARE']['BASE']['PRODUCT']
                    if product:
                        product_key = self.to_safe(
                            'HARDWARE-PRODUCT-%s' % asset['HARDWARE']['BASE']['PRODUCT'])
                        self.push(self.inventory, product_key, asset_identifier)

            # Indexing now complete, adds the host details to the asset cache.
            self.cache[asset_identifier] = asset

        try:
            self.write_to_cache(self.cache, self.cache_path_cache)
            self.write_to_cache(self.inventory, self.cache_path_inventory)
        except Exception:
            self.log.error("Error while writing to cache:\n%s", traceback.format_exc())
            return False
        return True

    def push(self, dictionary, key, value):
        """ Adds a value to a list at a dictionary key, creating the list if it doesn't
            exist. """

        if key not in dictionary:
            dictionary[key] = []
        dictionary[key].append(value)

    def get_host_info(self):
        """ Get variables about a specific host. """

        if not self.cache or len(self.cache) == 0:
            # Need to load index from cache
            self.load_cache_from_cache()

        if self.args.host not in self.cache:
            # try updating the cache
            self.update_cache()

            if self.args.host not in self.cache:
                # host might not exist anymore
                return self.json_format_dict({}, self.args.pretty)

        return self.json_format_dict(self.cache[self.args.host], self.args.pretty)

    def load_inventory_from_cache(self):
        """ Reads the index from the cache file sets self.index """

        try:
            cache = open(self.cache_path_inventory, 'r')
            json_inventory = cache.read()
            self.inventory = json.loads(json_inventory)
            return True
        except Exception:
            self.log.error("Error while loading inventory:\n%s",
                           traceback.format_exc())
            self.inventory = {}
            return False

    def load_cache_from_cache(self):
        """ Reads the cache from the cache file sets self.cache """

        try:
            cache = open(self.cache_path_cache, 'r')
            json_cache = cache.read()
            self.cache = json.loads(json_cache)
            return True
        except Exception:
            self.log.error("Error while loading host cache:\n%s",
                           traceback.format_exc())
            self.cache = {}
            return False

    def write_to_cache(self, data, filename):
        """ Writes data in JSON format to a specified file. """

        json_data = self.json_format_dict(data, self.args.pretty)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """ Converts 'bad' characters in a string to underscores so they
            can be used as Ansible groups """

        return re.sub(r"[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """ Converts a dict to a JSON object and dumps it as a formatted string """

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)


if __name__ in '__main__':
    inventory = CollinsInventory()
    if inventory.run():
        sys.exit(0)
    else:
        sys.exit(-1)