mirror of
https://github.com/ansible-collections/community.general.git
synced 2024-09-14 20:13:21 +02:00
Improve consul_io.py execution speed (#33737)
* Improve execution speed * fix missing whitespace
This commit is contained in:
parent
5bcc9cd1c3
commit
92c5b758f8
2 changed files with 59 additions and 10 deletions
|
@ -2,6 +2,12 @@
|
|||
|
||||
[consul]
|
||||
|
||||
#
|
||||
# Bulk load. Load all possible data before building inventory JSON
|
||||
# If true, script processes in-memory data. JSON generation reduces drastically
|
||||
#
|
||||
bulk_load = false
|
||||
|
||||
# restrict included nodes to those from this datacenter
|
||||
#datacenter = nyc1
|
||||
|
||||
|
@ -13,6 +19,14 @@ url = http://localhost:8500
|
|||
# a suffix of '_servers' will add each address to the group name 'redis_servers'
|
||||
servers_suffix = _servers
|
||||
|
||||
#
|
||||
# By default, final JSON is built based on all available info in consul.
|
||||
# Suffixes means that services groups will be added in addition to basic infromation. See servers_suffix for additional info
|
||||
# There are cases when speed is preferable than having services groups
|
||||
# False value will reduce script execution time dragtically.
|
||||
#
|
||||
suffixes = true
|
||||
|
||||
# if specified then the inventory will generate domain names that will resolve
|
||||
# via Consul's inbuilt DNS.
|
||||
#domain=consul
|
||||
|
@ -32,6 +46,9 @@ kv_metadata=ansible/metadata
|
|||
|
||||
# looks up the health of each service and adds the node to 'up' and 'down' groups
|
||||
# based on the service availibility
|
||||
#
|
||||
# !!!! if availability is true, suffixes also must be true. !!!!
|
||||
#
|
||||
availability = true
|
||||
available_suffix = _up
|
||||
unavailable_suffix = _down
|
||||
|
|
|
@ -218,6 +218,8 @@ class ConsulInventory(object):
|
|||
self.nodes_by_kv = {}
|
||||
self.nodes_by_availability = {}
|
||||
self.current_dc = None
|
||||
self.inmemory_kv = []
|
||||
self.inmemory_nodes = []
|
||||
|
||||
config = ConsulConfig()
|
||||
self.config = config
|
||||
|
@ -235,12 +237,21 @@ class ConsulInventory(object):
|
|||
self.combine_all_results()
|
||||
print(json.dumps(self.inventory, sort_keys=True, indent=2))
|
||||
|
||||
def bulk_load(self, datacenter):
|
||||
index, groups_list = self.consul_api.kv.get(self.config.kv_groups, recurse=True, dc=datacenter)
|
||||
index, metadata_list = self.consul_api.kv.get(self.config.kv_metadata, recurse=True, dc=datacenter)
|
||||
index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
|
||||
self.inmemory_kv += groups_list
|
||||
self.inmemory_kv += metadata_list
|
||||
self.inmemory_nodes += nodes
|
||||
|
||||
def load_all_data_consul(self):
|
||||
''' cycle through each of the datacenters in the consul catalog and process
|
||||
the nodes in each '''
|
||||
self.datacenters = self.consul_api.catalog.datacenters()
|
||||
for datacenter in self.datacenters:
|
||||
self.current_dc = datacenter
|
||||
self.bulk_load(datacenter)
|
||||
self.load_data_for_datacenter(datacenter)
|
||||
|
||||
def load_availability_groups(self, node, datacenter):
|
||||
|
@ -262,9 +273,20 @@ class ConsulInventory(object):
|
|||
self.add_node_to_map(self.nodes_by_availability,
|
||||
service_name + suffix, node['Node'])
|
||||
|
||||
def consul_get_kv_inmemory(self, key):
|
||||
result = filter(lambda x: x['Key'] == key, self.inmemory_kv)
|
||||
return result.pop() if result else None
|
||||
|
||||
def consul_get_node_inmemory(self, node):
|
||||
result = filter(lambda x: x['Node'] == node, self.inmemory_nodes)
|
||||
return {"Node": result.pop(), "Services": {}} if result else None
|
||||
|
||||
def load_data_for_datacenter(self, datacenter):
|
||||
'''processes all the nodes in a particular datacenter'''
|
||||
index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
|
||||
if self.config.bulk_load == 'true':
|
||||
nodes = self.inmemory_nodes
|
||||
else:
|
||||
index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
|
||||
for node in nodes:
|
||||
self.add_node_to_map(self.nodes_by_datacenter, datacenter, node)
|
||||
self.load_data_for_node(node['Node'], datacenter)
|
||||
|
@ -273,18 +295,22 @@ class ConsulInventory(object):
|
|||
'''loads the data for a sinle node adding it to various groups based on
|
||||
metadata retrieved from the kv store and service availability'''
|
||||
|
||||
index, node_data = self.consul_api.catalog.node(node, dc=datacenter)
|
||||
if self.config.suffixes == 'true':
|
||||
index, node_data = self.consul_api.catalog.node(node, dc=datacenter)
|
||||
else:
|
||||
node_data = self.consul_get_node_inmemory(node)
|
||||
node = node_data['Node']
|
||||
|
||||
self.add_node_to_map(self.nodes, 'all', node)
|
||||
self.add_metadata(node_data, "consul_datacenter", datacenter)
|
||||
self.add_metadata(node_data, "consul_nodename", node['Node'])
|
||||
|
||||
self.load_groups_from_kv(node_data)
|
||||
self.load_node_metadata_from_kv(node_data)
|
||||
self.load_availability_groups(node_data, datacenter)
|
||||
|
||||
for name, service in node_data['Services'].items():
|
||||
self.load_data_from_service(name, service, node_data)
|
||||
if self.config.suffixes == 'true':
|
||||
self.load_availability_groups(node_data, datacenter)
|
||||
for name, service in node_data['Services'].items():
|
||||
self.load_data_from_service(name, service, node_data)
|
||||
|
||||
def load_node_metadata_from_kv(self, node_data):
|
||||
''' load the json dict at the metadata path defined by the kv_metadata value
|
||||
|
@ -293,7 +319,10 @@ class ConsulInventory(object):
|
|||
node = node_data['Node']
|
||||
if self.config.has_config('kv_metadata'):
|
||||
key = "%s/%s/%s" % (self.config.kv_metadata, self.current_dc, node['Node'])
|
||||
index, metadata = self.consul_api.kv.get(key)
|
||||
if self.config.bulk_load == 'true':
|
||||
metadata = self.consul_get_kv_inmemory(key)
|
||||
else:
|
||||
index, metadata = self.consul_api.kv.get(key)
|
||||
if metadata and metadata['Value']:
|
||||
try:
|
||||
metadata = json.loads(metadata['Value'])
|
||||
|
@ -309,7 +338,10 @@ class ConsulInventory(object):
|
|||
node = node_data['Node']
|
||||
if self.config.has_config('kv_groups'):
|
||||
key = "%s/%s/%s" % (self.config.kv_groups, self.current_dc, node['Node'])
|
||||
index, groups = self.consul_api.kv.get(key)
|
||||
if self.config.bulk_load == 'true':
|
||||
groups = self.consul_get_kv_inmemory(key)
|
||||
else:
|
||||
index, groups = self.consul_api.kv.get(key)
|
||||
if groups and groups['Value']:
|
||||
for group in groups['Value'].split(','):
|
||||
self.add_node_to_map(self.nodes_by_kv, group.strip(), node)
|
||||
|
@ -434,11 +466,11 @@ class ConsulConfig(dict):
|
|||
config_options = ['host', 'token', 'datacenter', 'servers_suffix',
|
||||
'tags', 'kv_metadata', 'kv_groups', 'availability',
|
||||
'unavailable_suffix', 'available_suffix', 'url',
|
||||
'domain']
|
||||
'domain', 'suffixes', 'bulk_load']
|
||||
for option in config_options:
|
||||
value = None
|
||||
if config.has_option('consul', option):
|
||||
value = config.get('consul', option)
|
||||
value = config.get('consul', option).lower()
|
||||
setattr(self, option, value)
|
||||
|
||||
def read_cli_args(self):
|
||||
|
|
Loading…
Reference in a new issue