Fix python3 and nonascii handling in inventory plugins

Fixes #30663

lib/ansible/plugins/inventory/__init__.py

@@ -60,18 +60,18 @@ class BaseInventoryPlugin(object):
     def verify_file(self, path):
         ''' Verify if file is usable by this plugin, base does minimal accessability check '''
 
-        b_path = to_bytes(path)
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         return (os.path.exists(b_path) and os.access(b_path, os.R_OK))
 
     def get_cache_prefix(self, path):
         ''' create predictable unique prefix for plugin/inventory '''
 
         m = hashlib.sha1()
-        m.update(to_bytes(self.NAME))
+        m.update(to_bytes(self.NAME, errors='surrogate_or_strict'))
         d1 = m.hexdigest()
 
         n = hashlib.sha1()
-        n.update(to_bytes(path))
+        n.update(to_bytes(path, errors='surrogate_or_strict'))
         d2 = n.hexdigest()
 
         return 's_'.join([d1[:5], d2[:5]])

lib/ansible/plugins/inventory/advanced_host_list.py

@@ -36,9 +36,9 @@ class InventoryModule(BaseInventoryPlugin):
     def verify_file(self, host_list):
 
         valid = False
-        b_path = to_bytes(host_list)
+        b_path = to_bytes(host_list, errors='surrogate_or_strict')
         if not os.path.exists(b_path) and ',' in host_list:
-                valid = True
+            valid = True
         return valid
 
     def parse(self, inventory, loader, host_list, cache=True):
@@ -61,7 +61,7 @@ class InventoryModule(BaseInventoryPlugin):
                         if host not in self.inventory.hosts:
                             self.inventory.add_host(host, group='ungrouped', port=port)
         except Exception as e:
-            raise AnsibleParserError("Invalid data from string, could not parse: %s" % str(e))
+            raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e))
 
     def _expand_hostpattern(self, hostpattern):
         '''

lib/ansible/plugins/inventory/host_list.py

@@ -27,8 +27,7 @@ EXAMPLES = r'''
 import os
 
 from ansible.errors import AnsibleError, AnsibleParserError
-from ansible.module_utils.six import string_types
+from ansible.module_utils._text import to_bytes, to_native
-from ansible.module_utils._text import to_bytes, to_text, to_native
 from ansible.parsing.utils.addresses import parse_address
 from ansible.plugins.inventory import BaseInventoryPlugin
 
@@ -40,9 +39,9 @@ class InventoryModule(BaseInventoryPlugin):
     def verify_file(self, host_list):
 
         valid = False
-        b_path = to_bytes(host_list)
+        b_path = to_bytes(host_list, errors='surrogate_or_strict')
         if not os.path.exists(b_path) and ',' in host_list:
-                valid = True
+            valid = True
         return valid
 
     def parse(self, inventory, loader, host_list, cache=True):
@@ -64,4 +63,4 @@ class InventoryModule(BaseInventoryPlugin):
                     if host not in self.inventory.hosts:
                         self.inventory.add_host(host, group='ungrouped', port=port)
         except Exception as e:
-            raise AnsibleParserError("Invalid data from string, could not parse: %s" % str(e))
+            raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e))

lib/ansible/plugins/inventory/ini.py

@@ -105,7 +105,7 @@ class InventoryModule(BaseFileInventoryPlugin):
             if self.loader:
                 (b_data, private) = self.loader._get_file_contents(path)
             else:
-                b_path = to_bytes(path)
+                b_path = to_bytes(path, errors='surrogate_or_strict')
                 with open(b_path, 'rb') as fh:
                     b_data = fh.read()
 
@@ -366,14 +366,14 @@ class InventoryModule(BaseFileInventoryPlugin):
         # [naughty:children] # only get coal in their stockings
 
         self.patterns['section'] = re.compile(
-            r'''^\[
+            to_text(r'''^\[
                     ([^:\]\s]+)             # group name (see groupname below)
                     (?::(\w+))?             # optional : and tag name
                 \]
                 \s*                         # ignore trailing whitespace
                 (?:\#.*)?                   # and/or a comment till the
                 $                           # end of the line
-            ''', re.X
+            ''', errors='surrogate_or_strict'), re.X
         )
 
         # FIXME: What are the real restrictions on group names, or rather, what
@@ -382,10 +382,10 @@ class InventoryModule(BaseFileInventoryPlugin):
         # precise rules in order to support better diagnostics.
 
         self.patterns['groupname'] = re.compile(
-            r'''^
+            to_text(r'''^
                 ([^:\]\s]+)
                 \s*                         # ignore trailing whitespace
                 (?:\#.*)?                   # and/or a comment till the
                 $                           # end of the line
-            ''', re.X
+            ''', errors='surrogate_or_strict'), re.X
         )

lib/ansible/plugins/inventory/virtualbox.py

@@ -49,10 +49,11 @@ simple_config_file:
 
 import os
 
+from collections import MutableMapping
 from subprocess import Popen, PIPE
 
 from ansible.errors import AnsibleParserError
-from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.plugins.inventory import BaseInventoryPlugin
 
 
@@ -60,14 +61,14 @@ class InventoryModule(BaseInventoryPlugin):
     ''' Host inventory parser for ansible using local virtualbox. '''
 
     NAME = 'virtualbox'
-    VBOX = "VBoxManage"
+    VBOX = b"VBoxManage"
 
     def _query_vbox_data(self, host, property_path):
         ret = None
         try:
-            cmd = [self.VBOX, 'guestproperty', 'get', host, property_path]
+            cmd = [self.VBOX, b'guestproperty', b'get', to_bytes(host, errors='surrogate_or_strict'), to_bytes(property_path, errors='surrogate_or_strict')]
             x = Popen(cmd, stdout=PIPE)
-            ipinfo = x.stdout.read()
+            ipinfo = to_text(x.stdout.read(), errors='surrogate_or_strict')
             if 'Value' in ipinfo:
                 a, ip = ipinfo.split(':', 1)
                 ret = ip.strip()
@@ -81,7 +82,7 @@ class InventoryModule(BaseInventoryPlugin):
         for host in hostvars:
 
             # create vars from vbox properties
-            if data.get('query') and isinstance(data['query'], dict):
+            if data.get('query') and isinstance(data['query'], MutableMapping):
                 for varname in data['query']:
                     hostvars[host][varname] = self._query_vbox_data(host, data['query'][varname])
 
@@ -168,7 +169,7 @@ class InventoryModule(BaseInventoryPlugin):
         try:
             config_data = self.loader.load_from_file(path)
         except Exception as e:
-            raise AnsibleParserError(e)
+            raise AnsibleParserError(to_native(e))
 
         if not config_data or config_data.get('plugin') != self.NAME:
             # this is not my config file
@@ -182,26 +183,26 @@ class InventoryModule(BaseInventoryPlugin):
                 pass
 
         if not source_data:
-            pwfile = to_bytes(config_data.get('settings_password_file'))
+            b_pwfile = to_bytes(config_data.get('settings_password_file'), errors='surrogate_or_strict')
             running = config_data.get('running_only', False)
 
             # start getting data
-            cmd = [self.VBOX, 'list', '-l']
+            cmd = [self.VBOX, b'list', b'-l']
             if running:
-                cmd.append('runningvms')
+                cmd.append(b'runningvms')
             else:
-                cmd.append('vms')
+                cmd.append(b'vms')
 
-            if pwfile and os.path.exists(pwfile):
+            if b_pwfile and os.path.exists(b_pwfile):
-                cmd.append('--settingspwfile')
+                cmd.append(b'--settingspwfile')
-                cmd.append(pwfile)
+                cmd.append(b_pwfile)
 
             try:
                 p = Popen(cmd, stdout=PIPE)
             except Exception as e:
-                AnsibleParserError(e)
+                AnsibleParserError(to_native(e))
 
-            source_data = p.stdout.readlines()
+            source_data = p.stdout.read()
-            inventory.cache[cache_key] = to_text(source_data)
+            inventory.cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
 
-        self._populate_from_source(source_data, config_data)
+        self._populate_from_source(source_data.splitlines(), config_data)

lib/ansible/plugins/inventory/yaml.py

@@ -51,11 +51,12 @@ all: # keys must be unique, i.e. only one 'hosts' per group
 
 import re
 import os
+from collections import MutableMapping
 
 from ansible import constants as C
 from ansible.errors import AnsibleParserError
 from ansible.module_utils.six import string_types
-from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils._text import to_native
 from ansible.parsing.utils.addresses import parse_address
 from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range
 
@@ -74,9 +75,8 @@ class InventoryModule(BaseFileInventoryPlugin):
     def verify_file(self, path):
 
         valid = False
-        b_path = to_bytes(path)
+        if super(InventoryModule, self).verify_file(path):
-        if super(InventoryModule, self).verify_file(b_path):
+            file_name, ext = os.path.splitext(path)
-            file_name, ext = os.path.splitext(b_path)
             if ext and ext in C.YAML_FILENAME_EXTENSIONS:
                 valid = True
         return valid
@@ -96,11 +96,11 @@ class InventoryModule(BaseFileInventoryPlugin):
 
         # We expect top level keys to correspond to groups, iterate over them
         # to get host, vars and subgroups (which we iterate over recursivelly)
-        if isinstance(data, dict):
+        if isinstance(data, MutableMapping):
             for group_name in data:
                 self._parse_group(group_name, data[group_name])
         else:
-            raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % data)
+            raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(data))
 
     def _parse_group(self, group, group_data):
 
@@ -108,7 +108,7 @@ class InventoryModule(BaseFileInventoryPlugin):
 
             self.inventory.add_group(group)
 
-            if isinstance(group_data, dict):
+            if isinstance(group_data, MutableMapping):
                 # make sure they are dicts
                 for section in ['vars', 'children', 'hosts']:
                     if section in group_data and isinstance(group_data[section], string_types):
@@ -167,4 +167,4 @@ class InventoryModule(BaseFileInventoryPlugin):
         '''
         Compiles the regular expressions required to parse the inventory and stores them in self.patterns.
         '''
-        self.patterns['groupname'] = re.compile(r'''^[A-Za-z_][A-Za-z0-9_]*$''')
+        self.patterns['groupname'] = re.compile(u'''^[A-Za-z_][A-Za-z0-9_]*$''')