ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

PEP 8 E111 cleanup.

Browse source
This commit is contained in:
Matt Clay 2017-01-30 20:09:38 -08:00
parent 85300883ef
commit f80224f828
2 changed files with 237 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

474
contrib/inventory/consul_io.py
View file

@ -188,14 +188,14 @@ if os.getenv('ANSIBLE_INVENTORY_CONSUL_IO_LOG_ENABLED'):
try: try:
import json import json
except ImportError: except ImportError:
import simplejson as json import simplejson as json
try: try:
import consul import consul
except ImportError as e: except ImportError as e:
sys.exit("""failed=True msg='python-consul required for this module. sys.exit("""failed=True msg='python-consul required for this module.
See http://python-consul.readthedocs.org/en/latest/#installation'""") See http://python-consul.readthedocs.org/en/latest/#installation'""")
from six import iteritems from six import iteritems
@ -203,287 +203,287 @@ from six import iteritems
class ConsulInventory(object): class ConsulInventory(object):
def __init__(self): def __init__(self):
''' Create an inventory based on the catalog of nodes and services ''' Create an inventory based on the catalog of nodes and services
registered in a consul cluster''' registered in a consul cluster'''
self.node_metadata = {} self.node_metadata = {}
self.nodes = {} self.nodes = {}
self.nodes_by_service = {} self.nodes_by_service = {}
self.nodes_by_tag = {} self.nodes_by_tag = {}
self.nodes_by_datacenter = {} self.nodes_by_datacenter = {}
self.nodes_by_kv = {} self.nodes_by_kv = {}
self.nodes_by_availability = {} self.nodes_by_availability = {}
self.current_dc = None self.current_dc = None
config = ConsulConfig() config = ConsulConfig()
self.config = config self.config = config
self.consul_api = config.get_consul_api() self.consul_api = config.get_consul_api()
if config.has_config('datacenter'): if config.has_config('datacenter'):
if config.has_config('host'): if config.has_config('host'):
self.load_data_for_node(config.host, config.datacenter) self.load_data_for_node(config.host, config.datacenter)
else: else:
self.load_data_for_datacenter(config.datacenter) self.load_data_for_datacenter(config.datacenter)
else: else:
self.load_all_data_consul() self.load_all_data_consul()
self.combine_all_results() self.combine_all_results()
print(json.dumps(self.inventory, sort_keys=True, indent=2)) print(json.dumps(self.inventory, sort_keys=True, indent=2))
def load_all_data_consul(self): def load_all_data_consul(self):
''' cycle through each of the datacenters in the consul catalog and process ''' cycle through each of the datacenters in the consul catalog and process
the nodes in each ''' the nodes in each '''
self.datacenters = self.consul_api.catalog.datacenters() self.datacenters = self.consul_api.catalog.datacenters()
for datacenter in self.datacenters: for datacenter in self.datacenters:
self.current_dc = datacenter self.current_dc = datacenter
self.load_data_for_datacenter(datacenter) self.load_data_for_datacenter(datacenter)
def load_availability_groups(self, node, datacenter): def load_availability_groups(self, node, datacenter):
'''check the health of each service on a node and add add the node to either '''check the health of each service on a node and add add the node to either
an 'available' or 'unavailable' grouping. The suffix for each group can be an 'available' or 'unavailable' grouping. The suffix for each group can be
controlled from the config''' controlled from the config'''
if self.config.has_config('availability'): if self.config.has_config('availability'):
for service_name, service in iteritems(node['Services']): for service_name, service in iteritems(node['Services']):
for node in self.consul_api.health.service(service_name)[1]: for node in self.consul_api.health.service(service_name)[1]:
for check in node['Checks']: for check in node['Checks']:
if check['ServiceName'] == service_name: if check['ServiceName'] == service_name:
ok = 'passing' == check['Status'] ok = 'passing' == check['Status']
if ok: if ok:
suffix = self.config.get_availability_suffix( suffix = self.config.get_availability_suffix(
'available_suffix', '_available') 'available_suffix', '_available')
else: else:
suffix = self.config.get_availability_suffix( suffix = self.config.get_availability_suffix(
'unavailable_suffix', '_unavailable') 'unavailable_suffix', '_unavailable')
self.add_node_to_map(self.nodes_by_availability, self.add_node_to_map(self.nodes_by_availability,
service_name + suffix, node['Node']) service_name + suffix, node['Node'])
def load_data_for_datacenter(self, datacenter): def load_data_for_datacenter(self, datacenter):
'''processes all the nodes in a particular datacenter''' '''processes all the nodes in a particular datacenter'''
index, nodes = self.consul_api.catalog.nodes(dc=datacenter) index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
for node in nodes: for node in nodes:
self.add_node_to_map(self.nodes_by_datacenter, datacenter, node) self.add_node_to_map(self.nodes_by_datacenter, datacenter, node)
self.load_data_for_node(node['Node'], datacenter) self.load_data_for_node(node['Node'], datacenter)
def load_data_for_node(self, node, datacenter): def load_data_for_node(self, node, datacenter):
'''loads the data for a sinle node adding it to various groups based on '''loads the data for a sinle node adding it to various groups based on
metadata retrieved from the kv store and service availability''' metadata retrieved from the kv store and service availability'''
index, node_data = self.consul_api.catalog.node(node, dc=datacenter) index, node_data = self.consul_api.catalog.node(node, dc=datacenter)
node = node_data['Node'] node = node_data['Node']
self.add_node_to_map(self.nodes, 'all', node) self.add_node_to_map(self.nodes, 'all', node)
self.add_metadata(node_data, "consul_datacenter", datacenter) self.add_metadata(node_data, "consul_datacenter", datacenter)
self.add_metadata(node_data, "consul_nodename", node['Node']) self.add_metadata(node_data, "consul_nodename", node['Node'])
self.load_groups_from_kv(node_data) self.load_groups_from_kv(node_data)
self.load_node_metadata_from_kv(node_data) self.load_node_metadata_from_kv(node_data)
self.load_availability_groups(node_data, datacenter) self.load_availability_groups(node_data, datacenter)
for name, service in node_data['Services'].items(): for name, service in node_data['Services'].items():
self.load_data_from_service(name, service, node_data) self.load_data_from_service(name, service, node_data)
def load_node_metadata_from_kv(self, node_data): def load_node_metadata_from_kv(self, node_data):
''' load the json dict at the metadata path defined by the kv_metadata value ''' load the json dict at the metadata path defined by the kv_metadata value
and the node name add each entry in the dictionary to the the node's and the node name add each entry in the dictionary to the the node's
metadata ''' metadata '''
node = node_data['Node'] node = node_data['Node']
if self.config.has_config('kv_metadata'): if self.config.has_config('kv_metadata'):
key = "%s/%s/%s" % (self.config.kv_metadata, self.current_dc, node['Node']) key = "%s/%s/%s" % (self.config.kv_metadata, self.current_dc, node['Node'])
index, metadata = self.consul_api.kv.get(key) index, metadata = self.consul_api.kv.get(key)
if metadata and metadata['Value']: if metadata and metadata['Value']:
try: try:
metadata = json.loads(metadata['Value']) metadata = json.loads(metadata['Value'])
for k,v in metadata.items(): for k,v in metadata.items():
self.add_metadata(node_data, k, v) self.add_metadata(node_data, k, v)
except: except:
pass pass
def load_groups_from_kv(self, node_data): def load_groups_from_kv(self, node_data):
''' load the comma separated list of groups at the path defined by the ''' load the comma separated list of groups at the path defined by the
kv_groups config value and the node name add the node address to each kv_groups config value and the node name add the node address to each
group found ''' group found '''
node = node_data['Node'] node = node_data['Node']
if self.config.has_config('kv_groups'): if self.config.has_config('kv_groups'):
key = "%s/%s/%s" % (self.config.kv_groups, self.current_dc, node['Node']) key = "%s/%s/%s" % (self.config.kv_groups, self.current_dc, node['Node'])
index, groups = self.consul_api.kv.get(key) index, groups = self.consul_api.kv.get(key)
if groups and groups['Value']: if groups and groups['Value']:
for group in groups['Value'].split(','): for group in groups['Value'].split(','):
self.add_node_to_map(self.nodes_by_kv, group.strip(), node) self.add_node_to_map(self.nodes_by_kv, group.strip(), node)
def load_data_from_service(self, service_name, service, node_data): def load_data_from_service(self, service_name, service, node_data):
'''process a service registered on a node, adding the node to a group with '''process a service registered on a node, adding the node to a group with
the service name. Each service tag is extracted and the node is added to a the service name. Each service tag is extracted and the node is added to a
tag grouping also''' tag grouping also'''
self.add_metadata(node_data, "consul_services", service_name, True) self.add_metadata(node_data, "consul_services", service_name, True)
if self.is_service("ssh", service_name): if self.is_service("ssh", service_name):
self.add_metadata(node_data, "ansible_ssh_port", service['Port']) self.add_metadata(node_data, "ansible_ssh_port", service['Port'])
if self.config.has_config('servers_suffix'): if self.config.has_config('servers_suffix'):
service_name = service_name + self.config.servers_suffix service_name = service_name + self.config.servers_suffix
self.add_node_to_map(self.nodes_by_service, service_name, node_data['Node']) self.add_node_to_map(self.nodes_by_service, service_name, node_data['Node'])
self.extract_groups_from_tags(service_name, service, node_data) self.extract_groups_from_tags(service_name, service, node_data)
def is_service(self, target, name): def is_service(self, target, name):
return name and (name.lower() == target.lower()) return name and (name.lower() == target.lower())
def extract_groups_from_tags(self, service_name, service, node_data): def extract_groups_from_tags(self, service_name, service, node_data):
'''iterates each service tag and adds the node to groups derived from the '''iterates each service tag and adds the node to groups derived from the
service and tag names e.g. nginx_master''' service and tag names e.g. nginx_master'''
if self.config.has_config('tags') and service['Tags']: if self.config.has_config('tags') and service['Tags']:
tags = service['Tags'] tags = service['Tags']
self.add_metadata(node_data, "consul_%s_tags" % service_name, tags) self.add_metadata(node_data, "consul_%s_tags" % service_name, tags)
for tag in service['Tags']: for tag in service['Tags']:
tagname = service_name +'_'+tag tagname = service_name +'_'+tag
self.add_node_to_map(self.nodes_by_tag, tagname, node_data['Node']) self.add_node_to_map(self.nodes_by_tag, tagname, node_data['Node'])
def combine_all_results(self): def combine_all_results(self):
'''prunes and sorts all groupings for combination into the final map''' '''prunes and sorts all groupings for combination into the final map'''
self.inventory = {"_meta": { "hostvars" : self.node_metadata}} self.inventory = {"_meta": { "hostvars" : self.node_metadata}}
groupings = [self.nodes, self.nodes_by_datacenter, self.nodes_by_service, groupings = [self.nodes, self.nodes_by_datacenter, self.nodes_by_service,
self.nodes_by_tag, self.nodes_by_kv, self.nodes_by_availability] self.nodes_by_tag, self.nodes_by_kv, self.nodes_by_availability]
for grouping in groupings: for grouping in groupings:
for name, addresses in grouping.items(): for name, addresses in grouping.items():
self.inventory[name] = sorted(list(set(addresses))) self.inventory[name] = sorted(list(set(addresses)))
def add_metadata(self, node_data, key, value, is_list = False): def add_metadata(self, node_data, key, value, is_list = False):
''' Pushed an element onto a metadata dict for the node, creating ''' Pushed an element onto a metadata dict for the node, creating
the dict if it doesn't exist ''' the dict if it doesn't exist '''
key = self.to_safe(key) key = self.to_safe(key)
node = self.get_inventory_name(node_data['Node']) node = self.get_inventory_name(node_data['Node'])
if node in self.node_metadata: if node in self.node_metadata:
metadata = self.node_metadata[node] metadata = self.node_metadata[node]
else: else:
metadata = {} metadata = {}
self.node_metadata[node] = metadata self.node_metadata[node] = metadata
if is_list: if is_list:
self.push(metadata, key, value) self.push(metadata, key, value)
else: else:
metadata[key] = value metadata[key] = value
def get_inventory_name(self, node_data): def get_inventory_name(self, node_data):
'''return the ip or a node name that can be looked up in consul's dns''' '''return the ip or a node name that can be looked up in consul's dns'''
domain = self.config.domain domain = self.config.domain
if domain: if domain:
node_name = node_data['Node'] node_name = node_data['Node']
if self.current_dc: if self.current_dc:
return '%s.node.%s.%s' % ( node_name, self.current_dc, domain) return '%s.node.%s.%s' % ( node_name, self.current_dc, domain)
else: else:
return '%s.node.%s' % ( node_name, domain) return '%s.node.%s' % ( node_name, domain)
else: else:
return node_data['Address'] return node_data['Address']
def add_node_to_map(self, map, name, node): def add_node_to_map(self, map, name, node):
self.push(map, name, self.get_inventory_name(node)) self.push(map, name, self.get_inventory_name(node))
def push(self, my_dict, key, element): def push(self, my_dict, key, element):
''' Pushed an element onto an array that may not have been defined in the ''' Pushed an element onto an array that may not have been defined in the
dict ''' dict '''
key = self.to_safe(key) key = self.to_safe(key)
if key in my_dict: if key in my_dict:
my_dict[key].append(element) my_dict[key].append(element)
else: else:
my_dict[key] = [element] my_dict[key] = [element]
def to_safe(self, word): def to_safe(self, word):
''' Converts 'bad' characters in a string to underscores so they can be used ''' Converts 'bad' characters in a string to underscores so they can be used
as Ansible groups ''' as Ansible groups '''
return re.sub('[^A-Za-z0-9\-\.]', '_', word) return re.sub('[^A-Za-z0-9\-\.]', '_', word)
def sanitize_dict(self, d): def sanitize_dict(self, d):
new_dict = {} new_dict = {}
for k, v in d.items(): for k, v in d.items():
if v is not None: if v is not None:
new_dict[self.to_safe(str(k))] = self.to_safe(str(v)) new_dict[self.to_safe(str(k))] = self.to_safe(str(v))
return new_dict return new_dict
def sanitize_list(self, seq): def sanitize_list(self, seq):
new_seq = [] new_seq = []
for d in seq: for d in seq:
new_seq.append(self.sanitize_dict(d)) new_seq.append(self.sanitize_dict(d))
return new_seq return new_seq
class ConsulConfig(dict): class ConsulConfig(dict):
def __init__(self): def __init__(self):
self.read_settings() self.read_settings()
self.read_cli_args() self.read_cli_args()
def has_config(self, name): def has_config(self, name):
if hasattr(self, name): if hasattr(self, name):
return getattr(self, name) return getattr(self, name)
else: else:
return False return False
def read_settings(self): def read_settings(self):
''' Reads the settings from the consul.ini file ''' ''' Reads the settings from the consul.ini file '''
config = ConfigParser.SafeConfigParser() config = ConfigParser.SafeConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul.ini') config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul.ini')
config_options = ['host', 'token', 'datacenter', 'servers_suffix', config_options = ['host', 'token', 'datacenter', 'servers_suffix',
'tags', 'kv_metadata', 'kv_groups', 'availability', 'tags', 'kv_metadata', 'kv_groups', 'availability',
'unavailable_suffix', 'available_suffix', 'url', 'unavailable_suffix', 'available_suffix', 'url',
'domain'] 'domain']
for option in config_options: for option in config_options:
value = None value = None
if config.has_option('consul', option): if config.has_option('consul', option):
value = config.get('consul', option) value = config.get('consul', option)
setattr(self, option, value) setattr(self, option, value)
def read_cli_args(self): def read_cli_args(self):
''' Command line argument processing ''' ''' Command line argument processing '''
parser = argparse.ArgumentParser(description= parser = argparse.ArgumentParser(description=
'Produce an Ansible Inventory file based nodes in a Consul cluster') 'Produce an Ansible Inventory file based nodes in a Consul cluster')
parser.add_argument('--list', action='store_true', parser.add_argument('--list', action='store_true',
help='Get all inventory variables from all nodes in the consul cluster') help='Get all inventory variables from all nodes in the consul cluster')
parser.add_argument('--host', action='store', parser.add_argument('--host', action='store',
help='Get all inventory variables about a specific consul node, \ help='Get all inventory variables about a specific consul node, \
requires datacenter set in consul.ini.') requires datacenter set in consul.ini.')
parser.add_argument('--datacenter', action='store', parser.add_argument('--datacenter', action='store',
help='Get all inventory about a specific consul datacenter') help='Get all inventory about a specific consul datacenter')
args = parser.parse_args() args = parser.parse_args()
arg_names = ['host', 'datacenter'] arg_names = ['host', 'datacenter']
for arg in arg_names: for arg in arg_names:
if getattr(args, arg): if getattr(args, arg):
setattr(self, arg, getattr(args, arg)) setattr(self, arg, getattr(args, arg))
def get_availability_suffix(self, suffix, default): def get_availability_suffix(self, suffix, default):
if self.has_config(suffix): if self.has_config(suffix):
return self.has_config(suffix) return self.has_config(suffix)
return default return default
def get_consul_api(self): def get_consul_api(self):
'''get an instance of the api based on the supplied configuration''' '''get an instance of the api based on the supplied configuration'''
host = 'localhost' host = 'localhost'
port = 8500 port = 8500
token = None token = None
scheme = 'http' scheme = 'http'
if hasattr(self, 'url'): if hasattr(self, 'url'):
from urlparse import urlparse from urlparse import urlparse
o = urlparse(self.url) o = urlparse(self.url)
if o.hostname: if o.hostname:
host = o.hostname host = o.hostname
if o.port: if o.port:
port = o.port port = o.port
if o.scheme: if o.scheme:
scheme = o.scheme scheme = o.scheme
if hasattr(self, 'token'): if hasattr(self, 'token'):
token = self.token token = self.token
if not token: if not token:
token = 'anonymous' token = 'anonymous'
return consul.Consul(host=host, port=port, token=token, scheme=scheme) return consul.Consul(host=host, port=port, token=token, scheme=scheme)
ConsulInventory() ConsulInventory()

1
test/sanity/pep8/legacy-files.txt
View file

@ -1,5 +1,4 @@
contrib/inventory/abiquo.py contrib/inventory/abiquo.py
contrib/inventory/consul_io.py
contrib/inventory/digital_ocean.py contrib/inventory/digital_ocean.py
contrib/inventory/docker.py contrib/inventory/docker.py
contrib/inventory/ec2.py contrib/inventory/ec2.py