From 78ed9793fe398fa4bff273ab9a258ad8761089a8 Mon Sep 17 00:00:00 2001 From: Ner'zhul Date: Tue, 10 Jan 2017 15:09:11 +0100 Subject: [PATCH] Various VMWare inventory (pyvomi fixes) + ini doc (#19926) * Fix many points reported by PyCharm as PEP 8 code style * Improve inventory performance by dropping vim.HostSystem & vim.VirtualMachine collect when depth >= 2 * Declare some class variables properly * Remove some unused variables * Add documentation in vmware_inventory.ini for VMWARE_USERNAME & VMWARE_PASSWORD env vars --- contrib/inventory/vmware_inventory.ini | 20 +-- contrib/inventory/vmware_inventory.py | 185 ++++++++++++------------- 2 files changed, 101 insertions(+), 104 deletions(-) diff --git a/contrib/inventory/vmware_inventory.ini b/contrib/inventory/vmware_inventory.ini index d1b78d108c..713652e1f1 100644 --- a/contrib/inventory/vmware_inventory.ini +++ b/contrib/inventory/vmware_inventory.ini @@ -8,10 +8,12 @@ server=vcenter # The port for the vsphere API #port=443 -# The username with access to the vsphere API +# The username with access to the vsphere API. This setting +# may also be defined via the VMWARE_USERNAME environment variable. username=administrator@vsphere.local -# The password for the vsphere API +# The password for the vsphere API. This setting +# may also be defined via the VMWARE_PASSWORD environment variable. password=vmware # Verify the server's SSL certificate @@ -28,8 +30,8 @@ password=vmware # Max object level refers to the level of recursion the script will delve into -# the objects returned from pyvomi to find serializable facts. The default -# level of 0 is sufficient for most tasks and will be the most performant. +# the objects returned from pyvomi to find serializable facts. The default +# level of 0 is sufficient for most tasks and will be the most performant. # Beware that the recursion can exceed python's limit (causing traceback), # cause sluggish script performance and return huge blobs of facts. # If you do not know what you are doing, leave this set to 1. @@ -42,8 +44,8 @@ password=vmware # Host alias for objects in the inventory. VMWare allows duplicate VM names # so they can not be considered unique. Use this setting to alter the alias -# returned for the hosts. Any atributes for the guest can be used to build -# this alias. The default combines the config name and the config uuid and +# returned for the hosts. Any atributes for the guest can be used to build +# this alias. The default combines the config name and the config uuid and # expects that the ansible_host will be set by the host_pattern. #alias_pattern={{ config.name + '_' + config.uuid }} @@ -53,7 +55,7 @@ password=vmware #host_pattern={{ guest.ipaddress }} -# Host filters are a comma separated list of jinja patterns to remove +# Host filters are a comma separated list of jinja patterns to remove # non-matching hosts from the final result. # EXAMPLES: # host_filters={{ config.guestid == 'rhel7_64Guest' }} @@ -74,10 +76,10 @@ password=vmware # all available data. The serialization is comprehensive but slow. If the # vcenter environment is large and the desired properties are known, create # a 'properties' section in this config and make an arbitrary list of -# key=value settings where the value is a path to a specific property. If +# key=value settings where the value is a path to a specific property. If # If this feature is enabled, be sure to fetch every property that is used # in the jinja expressions defined above. For performance tuning, reduce -# the number of properties to the smallest amount possible and limit the +# the number of properties to the smallest amount possible and limit the # use of properties that are not direct attributes of vim.VirtualMachine #[properties] #prop01=name diff --git a/contrib/inventory/vmware_inventory.py b/contrib/inventory/vmware_inventory.py index a221293edc..548328f3c6 100755 --- a/contrib/inventory/vmware_inventory.py +++ b/contrib/inventory/vmware_inventory.py @@ -42,6 +42,7 @@ HAS_PYVMOMI = False try: from pyVmomi import vim from pyVim.connect import SmartConnect, Disconnect + HAS_PYVMOMI = True except ImportError: pass @@ -54,15 +55,17 @@ except ImportError: hasvcr = False try: import vcr + hasvcr = True except ImportError: pass + class VMwareMissingHostException(Exception): pass -class VMWareInventory(object): +class VMWareInventory(object): __name__ = 'VMWareInventory' guest_props = False @@ -76,14 +79,16 @@ class VMWareInventory(object): cache_max_age = None cache_path_cache = None cache_path_index = None + cache_dir = None server = None port = None username = None password = None + validate_certs = True host_filters = [] groupby_patterns = [] - if (sys.version_info > (3, 0)): + if sys.version_info > (3, 0): safe_types = [int, bool, str, float, None] else: safe_types = [int, long, bool, str, float, None] @@ -99,6 +104,11 @@ class VMWareInventory(object): 'parent', 'childtype'] + vimTableMaxDepth = { + "vim.HostSystem": 2, + "vim.VirtualMachine": 2, + } + # translation table for attributes to fetch for known vim types if not HAS_PYVMOMI: vimTable = {} @@ -106,14 +116,15 @@ class VMWareInventory(object): vimTable = { vim.Datastore: ['_moId', 'name'], vim.ResourcePool: ['_moId', 'name'], + vim.HostSystem: ['_moId', 'name'], } - def _empty_inventory(self): - return {"_meta" : {"hostvars" : {}}} - + @staticmethod + def _empty_inventory(): + return {"_meta": {"hostvars": {}}} def __init__(self, load=True): - self.inventory = self._empty_inventory() + self.inventory = VMWareInventory._empty_inventory() if load: # Read settings and parse CLI arguments @@ -135,7 +146,7 @@ class VMWareInventory(object): try: text = str(text) except UnicodeEncodeError: - text = text.encode('ascii','ignore') + text = text.encode('ascii', 'ignore') print('%s %s' % (datetime.datetime.now(), text)) def show(self): @@ -149,7 +160,6 @@ class VMWareInventory(object): data_to_print = self.inventory return json.dumps(data_to_print, indent=2) - def is_cache_valid(self): ''' Determines if the cache files have expired, or if it is still valid ''' @@ -164,7 +174,6 @@ class VMWareInventory(object): return valid - def do_api_calls_update_cache(self): ''' Get instances and cache the data ''' @@ -174,7 +183,6 @@ class VMWareInventory(object): self.inventory = self.instances_to_inventory(instances) self.write_to_cache(self.inventory, self.cache_path_cache) - def write_to_cache(self, data, cache_path): ''' Dump inventory to json file ''' @@ -182,7 +190,6 @@ class VMWareInventory(object): with open(self.cache_path_cache, 'wb') as f: f.write(json.dumps(data)) - def get_inventory_from_cache(self): ''' Read in jsonified inventory ''' @@ -192,7 +199,6 @@ class VMWareInventory(object): jdata = f.read() return json.loads(jdata) - def read_settings(self): ''' Reads the settings from the vmware_inventory.ini file ''' @@ -211,13 +217,13 @@ class VMWareInventory(object): 'cache_name': 'ansible-vmware', 'cache_path': '~/.ansible/tmp', 'cache_max_age': 3600, - 'max_object_level': 1, - 'alias_pattern': '{{ config.name + "_" + config.uuid }}', - 'host_pattern': '{{ guest.ipaddress }}', - 'host_filters': '{{ guest.gueststate == "running" }}', - 'groupby_patterns': '{{ guest.guestid }},{{ "templates" if config.template else "guests"}}', - 'lower_var_keys': True } - } + 'max_object_level': 1, + 'alias_pattern': '{{ config.name + "_" + config.uuid }}', + 'host_pattern': '{{ guest.ipaddress }}', + 'host_filters': '{{ guest.gueststate == "running" }}', + 'groupby_patterns': '{{ guest.guestid }},{{ "templates" if config.template else "guests"}}', + 'lower_var_keys': True} + } if six.PY3: config = configparser.ConfigParser() @@ -230,9 +236,9 @@ class VMWareInventory(object): config.read(vmware_ini_path) # apply defaults - for k,v in defaults['vmware'].items(): + for k, v in defaults['vmware'].items(): if not config.has_option('vmware', k): - config.set('vmware', k, str(v)) + config.set('vmware', k, str(v)) # where is the cache? self.cache_dir = os.path.expanduser(config.get('vmware', 'cache_path')) @@ -245,8 +251,8 @@ class VMWareInventory(object): self.debugl('cache path is %s' % self.cache_path_cache) self.cache_max_age = int(config.getint('vmware', 'cache_max_age')) - # mark the connection info - self.server = os.environ.get('VMWARE_SERVER', config.get('vmware', 'server')) + # mark the connection info + self.server = os.environ.get('VMWARE_SERVER', config.get('vmware', 'server')) self.debugl('server is %s' % self.server) self.port = int(os.environ.get('VMWARE_PORT', config.get('vmware', 'port'))) self.username = os.environ.get('VMWARE_USERNAME', config.get('vmware', 'username')) @@ -255,8 +261,7 @@ class VMWareInventory(object): self.validate_certs = os.environ.get('VMWARE_VALIDATE_CERTS', config.get('vmware', 'validate_certs')) if self.validate_certs in ['no', 'false', 'False', False]: self.validate_certs = False - else: - self.validate_certs = True + self.debugl('cert validation is %s' % self.validate_certs) # behavior control @@ -266,7 +271,7 @@ class VMWareInventory(object): if type(self.lowerkeys) != bool: if str(self.lowerkeys).lower() in ['yes', 'true', '1']: self.lowerkeys = True - else: + else: self.lowerkeys = False self.debugl('lower keys is %s' % self.lowerkeys) @@ -284,8 +289,7 @@ class VMWareInventory(object): self.guest_props.append(prop[1]) # save the config - self.config = config - + self.config = config def parse_cli_args(self): @@ -293,28 +297,24 @@ class VMWareInventory(object): parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on PyVmomi') parser.add_argument('--debug', action='store_true', default=False, - help='show debug info') + help='show debug info') parser.add_argument('--list', action='store_true', default=True, - help='List instances (default: True)') + help='List instances (default: True)') parser.add_argument('--host', action='store', - help='Get all the variables about a specific instance') + help='Get all the variables about a specific instance') parser.add_argument('--refresh-cache', action='store_true', default=False, - help='Force refresh of cache by making API requests to VSphere (default: False - use cache files)') + help='Force refresh of cache by making API requests to VSphere (default: False - use cache files)') parser.add_argument('--max-instances', default=None, type=int, - help='maximum number of instances to retrieve') + help='maximum number of instances to retrieve') self.args = parser.parse_args() - def get_instances(self): ''' Get a list of vm instances with pyvmomi ''' - - instances = [] - kwargs = {'host': self.server, 'user': self.username, 'pwd': self.password, - 'port': int(self.port) } + 'port': int(self.port)} if hasattr(ssl, 'SSLContext') and not self.validate_certs: context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) @@ -324,7 +324,6 @@ class VMWareInventory(object): instances = self._get_instances(kwargs) return instances - def _get_instances(self, inkwargs): ''' Make API calls ''' @@ -335,7 +334,7 @@ class VMWareInventory(object): self.debugl('retrieving all instances') if not si: print("Could not connect to the specified host using specified " - "username and password") + "username and password") return -1 atexit.register(Disconnect, si) content = si.RetrieveContent() @@ -350,7 +349,7 @@ class VMWareInventory(object): for child in children: # If requested, limit the total number of instances if self.args.max_instances: - if len(instances) >= (self.args.max_instances): + if len(instances) >= self.args.max_instances: break instances.append(child) self.debugl("%s total instances in container view" % len(instances)) @@ -358,9 +357,9 @@ class VMWareInventory(object): if self.args.host: instances = [x for x in instances if x.name == self.args.host] - instance_tuples = [] - for instance in sorted(instances): - if self.guest_props != False: + instance_tuples = [] + for instance in sorted(instances): + if self.guest_props: ifacts = self.facts_from_proplist(instance) else: ifacts = self.facts_from_vobj(instance) @@ -368,19 +367,15 @@ class VMWareInventory(object): self.debugl('facts collected for all instances') return instance_tuples - def instances_to_inventory(self, instances): ''' Convert a list of vm objects into a json compliant inventory ''' self.debugl('re-indexing instances based on ini settings') - inventory = self._empty_inventory() + inventory = VMWareInventory._empty_inventory() inventory['all'] = {} inventory['all']['hosts'] = [] - last_idata = None - total = len(instances) - for idx,instance in enumerate(instances): - + for idx, instance in enumerate(instances): # make a unique id for this object to avoid vmware's # numerous uuid's which aren't all unique. thisid = str(uuid.uuid4()) @@ -392,16 +387,19 @@ class VMWareInventory(object): inventory['_meta']['hostvars'][thisid]['ansible_uuid'] = thisid # Make a map of the uuid to the alias the user wants - name_mapping = self.create_template_mapping(inventory, - self.config.get('vmware', 'alias_pattern')) + name_mapping = self.create_template_mapping( + inventory, + self.config.get('vmware', 'alias_pattern') + ) # Make a map of the uuid to the ssh hostname the user wants - host_mapping = self.create_template_mapping(inventory, - self.config.get('vmware', 'host_pattern')) - + host_mapping = self.create_template_mapping( + inventory, + self.config.get('vmware', 'host_pattern') + ) # Reset the inventory keys - for k,v in name_mapping.items(): + for k, v in name_mapping.items(): if not host_mapping or not k in host_mapping: continue @@ -411,7 +409,7 @@ class VMWareInventory(object): inventory['_meta']['hostvars'][k]['ansible_host'] = host_mapping[k] # 1.9.x backwards compliance inventory['_meta']['hostvars'][k]['ansible_ssh_host'] = host_mapping[k] - except Exception as e: + except Exception: continue if k == v: @@ -434,7 +432,7 @@ class VMWareInventory(object): continue self.debugl('filter: %s' % hf) filter_map = self.create_template_mapping(inventory, hf, dtype='boolean') - for k,v in filter_map.items(): + for k, v in filter_map.items(): if not v: # delete this host inventory['all']['hosts'].remove(k) @@ -447,25 +445,24 @@ class VMWareInventory(object): # Create groups for gbp in self.groupby_patterns: groupby_map = self.create_template_mapping(inventory, gbp) - for k,v in groupby_map.items(): + for k, v in groupby_map.items(): if v not in inventory: inventory[v] = {} inventory[v]['hosts'] = [] if k not in inventory[v]['hosts']: - inventory[v]['hosts'].append(k) + inventory[v]['hosts'].append(k) return inventory - def create_template_mapping(self, inventory, pattern, dtype='string'): ''' Return a hash of uuid to templated string from pattern ''' mapping = {} - for k,v in inventory['_meta']['hostvars'].items(): + for k, v in inventory['_meta']['hostvars'].items(): t = jinja2.Template(pattern) newkey = None - try: + try: newkey = t.render(v) newkey = newkey.strip() except Exception as e: @@ -478,9 +475,9 @@ class VMWareInventory(object): if newkey.lower() == 'false': newkey = False elif newkey.lower() == 'true': - newkey = True + newkey = True elif dtype == 'string': - pass + pass mapping[k] = newkey return mapping @@ -494,7 +491,7 @@ class VMWareInventory(object): if self.lowerkeys: key = key.lower() - if not '.' in prop: + if '.' not in prop: # props without periods are direct attributes of the parent rdata[key] = getattr(vm, prop) else: @@ -507,7 +504,7 @@ class VMWareInventory(object): # pointer to the current result key lastref = rdata - for idx,x in enumerate(parts): + for idx, x in enumerate(parts): # if the val wasn't set yet, get it from the parent if not val: @@ -533,7 +530,6 @@ class VMWareInventory(object): return rdata - def facts_from_vobj(self, vobj, level=0): ''' Traverse a VM object and return a json compliant data structure ''' @@ -556,7 +552,7 @@ class VMWareInventory(object): methods = dir(vobj) methods = [str(x) for x in methods if not x.startswith('_')] - methods = [x for x in methods if not x in self.bad_types] + methods = [x for x in methods if x not in self.bad_types] methods = [x for x in methods if not x.lower() in self.skip_keys] methods = sorted(methods) @@ -575,21 +571,22 @@ class VMWareInventory(object): method = method.lower() rdata[method] = self._process_object_types( - methodToCall, - thisvm=vobj, - inkey=method - ) + methodToCall, + thisvm=vobj, + inkey=method, + ) return rdata - def _process_object_types(self, vobj, thisvm=None, inkey=None, level=0): ''' Serialize an object ''' rdata = {} + if type(vobj).__name__ in self.vimTableMaxDepth and level >= self.vimTableMaxDepth[type(vobj).__name__]: + return rdata + if vobj is None: rdata = None - elif type(vobj) in self.vimTable: rdata = {} for key in self.vimTable[type(vobj)]: @@ -612,19 +609,17 @@ class VMWareInventory(object): rdata = [] try: vobj = sorted(vobj) - except Exception as e: + except Exception: pass for idv, vii in enumerate(vobj): - - if (level+1 <= self.maxlevel): - + if level + 1 <= self.maxlevel: vid = self._process_object_types( - vii, - thisvm=thisvm, - inkey=inkey+'['+str(idv)+']', - level=(level+1) - ) + vii, + thisvm=thisvm, + inkey=inkey + '[' + str(idv) + ']', + level=(level + 1) + ) if vid: rdata.append(vid) @@ -635,7 +630,7 @@ class VMWareInventory(object): elif issubclass(type(vobj), object): methods = dir(vobj) methods = [str(x) for x in methods if not x.startswith('_')] - methods = [x for x in methods if not x in self.bad_types] + methods = [x for x in methods if x not in self.bad_types] methods = [x for x in methods if not x.lower() in self.skip_keys] methods = sorted(methods) @@ -645,33 +640,33 @@ class VMWareInventory(object): methodToCall = getattr(vobj, method) except Exception as e: continue + if callable(methodToCall): continue + if self.lowerkeys: method = method.lower() - if (level+1 <= self.maxlevel): + if level + 1 <= self.maxlevel: rdata[method] = self._process_object_types( - methodToCall, - thisvm=thisvm, - inkey=inkey+'.'+method, - level=(level+1) - ) + methodToCall, + thisvm=thisvm, + inkey=inkey + '.' + method, + level=(level + 1) + ) else: pass return rdata def get_host_info(self, host): - + ''' Return hostvars for a single host ''' if host in self.inventory['_meta']['hostvars']: return self.inventory['_meta']['hostvars'][host] elif self.args.host and self.inventory['_meta']['hostvars']: - # check if the machine has the name requested - keys = self.inventory['_meta']['hostvars'].keys() match = None - for k,v in self.inventory['_meta']['hostvars'].items(): + for k, v in self.inventory['_meta']['hostvars']: if self.inventory['_meta']['hostvars'][k]['name'] == self.args.host: match = k break