From 4ed88512e45112f9670560ac3f01707a40a7f5c5 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 6 Sep 2016 22:54:17 -0700 Subject: [PATCH] Move uses of to_bytes, to_text, to_native to use the module_utils version (#17423) We couldn't copy to_unicode, to_bytes, to_str into module_utils because of licensing. So once created it we had two sets of functions that did the same things but had different implementations. To remedy that, this change removes the ansible.utils.unicode versions of those functions. --- bin/ansible | 12 +- docsite/rst/developing_modules_python3.rst | 22 ++ hacking/module_formatter.py | 5 +- lib/ansible/cli/__init__.py | 4 +- lib/ansible/cli/adhoc.py | 6 +- lib/ansible/cli/console.py | 25 +- lib/ansible/cli/galaxy.py | 19 +- lib/ansible/cli/vault.py | 4 +- lib/ansible/errors/__init__.py | 13 +- lib/ansible/executor/module_common.py | 42 ++- lib/ansible/executor/playbook_executor.py | 18 +- lib/ansible/executor/process/worker.py | 19 +- lib/ansible/executor/task_executor.py | 37 ++- lib/ansible/executor/task_queue_manager.py | 22 +- lib/ansible/galaxy/api.py | 10 +- lib/ansible/inventory/__init__.py | 47 +-- lib/ansible/inventory/ini.py | 10 +- lib/ansible/inventory/script.py | 12 +- lib/ansible/module_utils/_text.py | 24 +- lib/ansible/module_utils/basic.py | 14 +- lib/ansible/parsing/dataloader.py | 61 ++-- lib/ansible/parsing/splitter.py | 19 +- lib/ansible/parsing/vault/__init__.py | 34 +- lib/ansible/parsing/yaml/constructor.py | 8 +- lib/ansible/parsing/yaml/objects.py | 3 +- lib/ansible/playbook/base.py | 24 +- lib/ansible/playbook/task.py | 26 +- lib/ansible/plugins/__init__.py | 29 +- lib/ansible/plugins/action/__init__.py | 62 ++-- lib/ansible/plugins/action/assemble.py | 6 +- lib/ansible/plugins/action/async.py | 9 +- lib/ansible/plugins/action/copy.py | 20 +- lib/ansible/plugins/action/debug.py | 4 +- lib/ansible/plugins/action/fetch.py | 4 +- lib/ansible/plugins/action/include_vars.py | 4 +- lib/ansible/plugins/action/net_config.py | 9 +- lib/ansible/plugins/action/net_template.py | 10 +- lib/ansible/plugins/action/patch.py | 6 +- lib/ansible/plugins/action/script.py | 7 +- lib/ansible/plugins/action/template.py | 13 +- lib/ansible/plugins/action/unarchive.py | 7 +- lib/ansible/plugins/action/win_reboot.py | 17 +- lib/ansible/plugins/cache/jsonfile.py | 2 +- lib/ansible/plugins/callback/__init__.py | 20 +- lib/ansible/plugins/callback/junit.py | 5 +- lib/ansible/plugins/callback/log_plays.py | 3 +- lib/ansible/plugins/callback/mail.py | 5 +- lib/ansible/plugins/callback/tree.py | 5 +- lib/ansible/plugins/connection/__init__.py | 8 +- lib/ansible/plugins/connection/accelerate.py | 7 +- lib/ansible/plugins/connection/chroot.py | 11 +- lib/ansible/plugins/connection/docker.py | 13 +- lib/ansible/plugins/connection/jail.py | 9 +- lib/ansible/plugins/connection/libvirt_lxc.py | 9 +- lib/ansible/plugins/connection/local.py | 13 +- lib/ansible/plugins/connection/lxc.py | 22 +- lib/ansible/plugins/connection/lxd.py | 16 +- .../plugins/connection/paramiko_ssh.py | 11 +- lib/ansible/plugins/connection/ssh.py | 24 +- lib/ansible/plugins/connection/winrm.py | 64 ++-- lib/ansible/plugins/connection/zone.py | 3 +- lib/ansible/plugins/filter/core.py | 26 +- lib/ansible/plugins/lookup/__init__.py | 4 +- lib/ansible/plugins/lookup/csvfile.py | 7 +- lib/ansible/plugins/lookup/fileglob.py | 7 +- lib/ansible/plugins/lookup/filetree.py | 17 +- lib/ansible/plugins/lookup/ini.py | 8 +- lib/ansible/plugins/lookup/shelvefile.py | 6 +- lib/ansible/plugins/lookup/template.py | 6 +- lib/ansible/plugins/lookup/url.py | 10 +- lib/ansible/plugins/shell/powershell.py | 13 +- lib/ansible/plugins/strategy/__init__.py | 23 +- lib/ansible/plugins/strategy/free.py | 15 +- lib/ansible/plugins/strategy/linear.py | 7 +- lib/ansible/template/__init__.py | 40 +-- lib/ansible/template/vars.py | 6 +- lib/ansible/utils/display.py | 14 +- lib/ansible/utils/hashing.py | 8 +- lib/ansible/utils/path.py | 14 +- lib/ansible/utils/shlex.py | 5 +- lib/ansible/utils/unicode.py | 296 +++--------------- lib/ansible/utils/vars.py | 12 +- lib/ansible/vars/unsafe_proxy.py | 13 +- test/units/mock/procenv.py | 5 +- test/units/parsing/vault/test_vault.py | 3 +- test/units/parsing/vault/test_vault_editor.py | 28 +- test/units/parsing/yaml/test_loader.py | 4 +- test/units/plugins/action/test_action.py | 18 +- .../connections/test_connection_ssh.py | 41 +-- 89 files changed, 759 insertions(+), 894 deletions(-) diff --git a/bin/ansible b/bin/ansible index be4edc2308..c7abff0180 100755 --- a/bin/ansible +++ b/bin/ansible @@ -43,7 +43,7 @@ from multiprocessing import Lock import ansible.constants as C from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.utils.display import Display -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text ######################################## @@ -97,10 +97,10 @@ if __name__ == '__main__': except AnsibleOptionsError as e: cli.parser.print_help() - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) exit_code = 5 except AnsibleParserError as e: - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) exit_code = 4 # TQM takes care of these, but leaving comment to reserve the exit codes # except AnsibleHostUnreachable as e: @@ -110,16 +110,16 @@ if __name__ == '__main__': # display.error(str(e)) # exit_code = 2 except AnsibleError as e: - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) exit_code = 1 except KeyboardInterrupt: display.error("User interrupted execution") exit_code = 99 except Exception as e: have_cli_options = cli is not None and cli.options is not None - display.error("Unexpected Exception: %s" % to_unicode(e), wrap_text=False) + display.error("Unexpected Exception: %s" % to_text(e), wrap_text=False) if not have_cli_options or have_cli_options and cli.options.verbosity > 2: - display.display(u"the full traceback was:\n\n%s" % to_unicode(traceback.format_exc())) + display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc())) else: display.display("to see the full traceback, use -vvv") exit_code = 250 diff --git a/docsite/rst/developing_modules_python3.rst b/docsite/rst/developing_modules_python3.rst index 26fdc9ad7f..901dfaed17 100644 --- a/docsite/rst/developing_modules_python3.rst +++ b/docsite/rst/developing_modules_python3.rst @@ -215,6 +215,28 @@ Python3. We'll need to gather experience to see if this is going to work out well for modules as well or if we should give the module_utils API explicit switches so that modules can choose to operate with text type all of the time. +Helpers +~~~~~~~ + +For converting between bytes, text, and native strings we have three helper +functions. These are :func:`ansible.module_utils._text.to_bytes`, +:func:`ansible.module_utils._text.to_native`, and +:func:`ansible.module_utils._text.to_text`. These are similar to using +``bytes.decode()`` and ``unicode.encode()`` with a few differences. + +* By default they try very hard not to traceback. +* The default encoding is "utf-8" +* There are two error strategies that don't correspond one-to-one with + a python codec error handler. These are ``surrogate_or_strict`` and + ``surrogate_or_replace``. ``surrogate_or_strict`` will use the ``surrogateescape`` + error handler if available (mostly on python3) or strict if not. It is most + appropriate to use when dealing with something that needs to round trip its + value like file paths database keys, etc. Without ``surrogateescape`` the best + thing these values can do is generate a traceback that our code can catch + and decide how to show an error message. ``surrogate_or_replace`` is for + when a value is going to be displayed to the user. If the + ``surrogateescape`` error handler is not present, it will replace + undecodable byte sequences with a replacement character. ================================ Porting Core Ansible to Python 3 diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 7a4005c044..6e84cada28 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -19,6 +19,7 @@ # from __future__ import print_function +__metaclass__ = type import os import glob @@ -34,10 +35,10 @@ from collections import defaultdict from jinja2 import Environment, FileSystemLoader from six import iteritems +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes from ansible.utils import module_docs from ansible.utils.vars import merge_hash -from ansible.utils.unicode import to_bytes -from ansible.errors import AnsibleError ##################################################################################### # constants and paths diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index b12fc0dbc6..3176d9c956 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -33,7 +33,7 @@ import subprocess from ansible.release import __version__ from ansible import constants as C from ansible.errors import AnsibleError, AnsibleOptionsError -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text try: from __main__ import display @@ -109,7 +109,7 @@ class CLI(object): if self.options.verbosity > 0: if C.CONFIG_FILE: - display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE)) + display.display(u"Using %s as config file" % to_text(C.CONFIG_FILE)) else: display.display(u"No config file found; using defaults") diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py index cfaeca955b..95ab640ded 100644 --- a/lib/ansible/cli/adhoc.py +++ b/lib/ansible/cli/adhoc.py @@ -27,13 +27,13 @@ from ansible.cli import CLI from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.executor.task_queue_manager import TaskQueueManager from ansible.inventory import Inventory +from ansible.module_utils._text import to_text from ansible.parsing.dataloader import DataLoader from ansible.parsing.splitter import parse_kv from ansible.playbook.play import Play from ansible.plugins import get_all_plugin_loaders from ansible.utils.vars import load_extra_vars from ansible.utils.vars import load_options_vars -from ansible.utils.unicode import to_unicode from ansible.vars import VariableManager try: @@ -99,7 +99,7 @@ class AdHocCLI(CLI): super(AdHocCLI, self).run() # only thing left should be host pattern - pattern = to_unicode(self.args[0], errors='strict') + pattern = to_text(self.args[0], errors='surrogate_or_strict') # ignore connection password cause we are local if self.options.connection == "local": @@ -169,7 +169,7 @@ class AdHocCLI(CLI): play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) - if self.callback: + if self.callback: cb = self.callback elif self.options.one_line: cb = 'oneline' diff --git a/lib/ansible/cli/console.py b/lib/ansible/cli/console.py index 5b0432036f..d2a1d1252f 100644 --- a/lib/ansible/cli/console.py +++ b/lib/ansible/cli/console.py @@ -39,18 +39,16 @@ import sys from ansible import constants as C from ansible.cli import CLI from ansible.errors import AnsibleError - from ansible.executor.task_queue_manager import TaskQueueManager from ansible.inventory import Inventory +from ansible.module_utils._text import to_native, to_text from ansible.parsing.dataloader import DataLoader from ansible.parsing.splitter import parse_kv from ansible.playbook.play import Play -from ansible.vars import VariableManager +from ansible.plugins import module_loader from ansible.utils import module_docs from ansible.utils.color import stringc -from ansible.utils.unicode import to_unicode, to_str -from ansible.plugins import module_loader - +from ansible.vars import VariableManager try: from __main__ import display @@ -152,11 +150,11 @@ class ConsoleCLI(CLI, cmd.Cmd): continue elif module.startswith('_'): fullpath = '/'.join([path,module]) - if os.path.islink(fullpath): # avoids aliases + if os.path.islink(fullpath): # avoids aliases continue module = module.replace('_', '', 1) - module = os.path.splitext(module)[0] # removes the extension + module = os.path.splitext(module)[0] # removes the extension yield module def default(self, arg, forceshell=False): @@ -192,11 +190,11 @@ class ConsoleCLI(CLI, cmd.Cmd): ) play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader) except Exception as e: - display.error(u"Unable to build command: %s" % to_unicode(e)) + display.error(u"Unable to build command: %s" % to_text(e)) return False try: - cb = 'minimal' #FIXME: make callbacks configurable + cb = 'minimal' # FIXME: make callbacks configurable # now create a task queue manager to execute the play self._tqm = None try: @@ -225,8 +223,8 @@ class ConsoleCLI(CLI, cmd.Cmd): display.error('User interrupted execution') return False except Exception as e: - display.error(to_unicode(e)) - #FIXME: add traceback in very very verbose mode + display.error(to_text(e)) + # FIXME: add traceback in very very verbose mode return False def emptyline(self): @@ -379,7 +377,7 @@ class ConsoleCLI(CLI, cmd.Cmd): else: completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)] - return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))] + return [to_native(s)[offs:] for s in completions if to_native(s).startswith(to_native(mline))] def completedefault(self, text, line, begidx, endidx): if line.split()[0] in self.modules: @@ -394,7 +392,6 @@ class ConsoleCLI(CLI, cmd.Cmd): oc, a, _ = module_docs.get_docstring(in_path) return oc['options'].keys() - def run(self): super(ConsoleCLI, self).run() @@ -410,7 +407,6 @@ class ConsoleCLI(CLI, cmd.Cmd): self.pattern = self.args[0] self.options.cwd = self.pattern - # dynamically add modules as commands self.modules = self.list_modules() for module in self.modules: @@ -465,4 +461,3 @@ class ConsoleCLI(CLI, cmd.Cmd): atexit.register(readline.write_history_file, histfile) self.set_prompt() self.cmdloop() - diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index 231b6b909a..457e15d5c1 100644 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -39,7 +39,7 @@ from ansible.galaxy.role import GalaxyRole from ansible.galaxy.login import GalaxyLogin from ansible.galaxy.token import GalaxyToken from ansible.playbook.role.requirement import RoleRequirement -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text try: from __main__ import display @@ -47,6 +47,7 @@ except ImportError: from ansible.utils.display import Display display = Display() + class GalaxyCLI(CLI): SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" ) @@ -65,7 +66,6 @@ class GalaxyCLI(CLI): epilog = "\nSee '%s --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0]) ) - self.set_action() # common @@ -111,7 +111,7 @@ class GalaxyCLI(CLI): if self.action in ['init', 'info']: self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles") - if not self.action in ("delete","import","init","login","setup"): + if self.action not in ("delete","import","init","login","setup"): # NOTE: while the option type=str, the default is a list, and the # callback will set the value to a list. self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH, @@ -142,7 +142,7 @@ class GalaxyCLI(CLI): def _display_role_info(self, role_info): - text = [u"", u"Role: %s" % to_unicode(role_info['name'])] + text = [u"", u"Role: %s" % to_text(role_info['name'])] text.append(u"\tdescription: %s" % role_info.get('description', '')) for k in sorted(role_info.keys()): @@ -340,7 +340,7 @@ class GalaxyCLI(CLI): f = open(role_file, 'r') if role_file.endswith('.yaml') or role_file.endswith('.yml'): try: - required_roles = yaml.safe_load(f.read()) + required_roles = yaml.safe_load(f.read()) except Exception as e: raise AnsibleError("Unable to load data from the requirements file: %s" % role_file) @@ -502,7 +502,7 @@ class GalaxyCLI(CLI): if len(self.args): terms = [] for i in range(len(self.args)): - terms.append(self.args.pop()) + terms.append(self.args.pop()) search = '+'.join(terms[::-1]) if not search and not self.options.platforms and not self.options.tags and not self.options.author: @@ -578,8 +578,8 @@ class GalaxyCLI(CLI): if len(self.args) < 2: raise AnsibleError("Expected a github_username and github_repository. Use --help.") - github_repo = self.args.pop() - github_user = self.args.pop() + github_repo = to_text(self.args.pop(), errors='surrogate_or_strict') + github_user = to_text(self.args.pop(), errors='surrogate_or_strict') if self.options.check_status: task = self.api.get_import_task(github_user=github_user, github_repo=github_repo) @@ -594,7 +594,8 @@ class GalaxyCLI(CLI): display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED) for t in task: display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED) - display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED) + display.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo), + color=C.COLOR_CHANGED) return 0 # found a single role as expected display.display("Successfully submitted import request %d" % task[0]['id']) diff --git a/lib/ansible/cli/vault.py b/lib/ansible/cli/vault.py index 85bdac6993..65657ca70e 100644 --- a/lib/ansible/cli/vault.py +++ b/lib/ansible/cli/vault.py @@ -26,7 +26,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.parsing.dataloader import DataLoader from ansible.parsing.vault import VaultEditor from ansible.cli import CLI -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text try: from __main__ import display @@ -163,7 +163,7 @@ class VaultCLI(CLI): # unicode here because we are displaying it and therefore can make # the decision that the display doesn't have to be precisely what # the input was (leave that to decrypt instead) - self.pager(to_unicode(self.editor.plaintext(f))) + self.pager(ansible.module_utils._text.to_text(self.editor.plaintext(f))) def execute_rekey(self): for f in self.args: diff --git a/lib/ansible/errors/__init__.py b/lib/ansible/errors/__init__.py index 7f02a44bc0..2186e6bc74 100644 --- a/lib/ansible/errors/__init__.py +++ b/lib/ansible/errors/__init__.py @@ -25,8 +25,7 @@ from ansible.errors.yaml_strings import ( YAML_POSITION_DETAILS, YAML_COMMON_UNQUOTED_COLON_ERROR, YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR, YAML_COMMON_UNBALANCED_QUOTES_ERROR ) - -from ansible.utils.unicode import to_unicode, to_str +from ansible.module_utils._text import to_native, to_text class AnsibleError(Exception): @@ -54,11 +53,11 @@ class AnsibleError(Exception): if obj and isinstance(obj, AnsibleBaseYAMLObject): extended_error = self._get_extended_error() if extended_error and not suppress_extended_error: - self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error)) + self.message = '%s\n\n%s' % (to_native(message), to_native(extended_error)) else: - self.message = '%s' % to_str(message) + self.message = '%s' % to_native(message) else: - self.message = '%s' % to_str(message) + self.message = '%s' % to_native(message) def __str__(self): return self.message @@ -104,8 +103,8 @@ class AnsibleError(Exception): error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number) if src_file not in ('', '') and self._show_content: (target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1) - target_line = to_unicode(target_line) - prev_line = to_unicode(prev_line) + target_line = to_text(target_line) + prev_line = to_text(prev_line) if target_line: stripped_line = target_line.replace(" ","") arrow_line = (" " * (col_number-1)) + "^ here" diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index 64a6d88283..86549ea0b7 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -29,11 +29,10 @@ import shlex import zipfile from io import BytesIO -# from Ansible from ansible.release import __version__, __author__ from ansible import constants as C from ansible.errors import AnsibleError -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text # Must import strategy and use write_locks from there # If we import write_locks directly then we end up binding a # variable to the object and then it never gets updated. @@ -45,6 +44,7 @@ except ImportError: from ansible.utils.display import Display display = Display() + REPLACER = b"#<>" REPLACER_VERSION = b"\"<>\"" REPLACER_COMPLEX = b"\"<>\"" @@ -239,7 +239,9 @@ def debug(command, zipped_mod, json_params): else: os.environ['PYTHONPATH'] = basedir - p = subprocess.Popen([%(interpreter)s, script_path, args_path], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([%(interpreter)s, script_path, args_path], + env=os.environ, shell=False, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE) (stdout, stderr) = p.communicate() if not isinstance(stderr, (bytes, unicode)): @@ -328,6 +330,7 @@ if __name__ == '__main__': sys.exit(exitcode) ''' + def _strip_comments(source): # Strip comments and blank lines from the wrapper buf = [] @@ -338,6 +341,7 @@ def _strip_comments(source): buf.append(line) return u'\n'.join(buf) + if C.DEFAULT_KEEP_REMOTE_FILES: # Keep comments when KEEP_REMOTE_FILES is set. That way users will see # the comments with some nice usage instructions @@ -346,6 +350,7 @@ else: # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE) + class ModuleDepFinder(ast.NodeVisitor): # Caveats: # This code currently does not handle: @@ -404,6 +409,7 @@ def _slurp(path): fd.close() return data + def _get_shebang(interpreter, task_vars, args=tuple()): """ Note not stellar API: @@ -425,6 +431,7 @@ def _get_shebang(interpreter, task_vars, args=tuple()): return (shebang, interpreter) + def recursive_finder(name, data, py_module_names, py_module_cache, zf): """ Using ModuleDepFinder, make sure we have all of the module_utils files that @@ -529,11 +536,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): # Save memory; the file won't have to be read again for this ansible module. del py_module_cache[py_module_file] + def _is_binary(module_data): textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) start = module_data[:1024] return bool(start.translate(None, textchars)) + def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): """ Given the source of the module, convert it to a Jinja2 template to insert @@ -617,9 +626,12 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # Create the module zip data zipoutput = BytesIO() zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) - ### Note: If we need to import from release.py first, - ### remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 - zf.writestr('ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n') + # Note: If we need to import from release.py first, + # remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 + zf.writestr('ansible/__init__.py', + b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + + to_bytes(__version__) + b'"\n__author__="' + + to_bytes(__author__) + b'"\n') zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n') zf.writestr('ansible_module_%s.py' % module_name, module_data) @@ -655,8 +667,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta try: zipdata = open(cached_module_filename, 'rb').read() except IOError: - raise AnsibleError('A different worker process failed to create module file. Look at traceback for that process for debugging information.') - zipdata = to_unicode(zipdata, errors='strict') + raise AnsibleError('A different worker process failed to create module file.' + ' Look at traceback for that process for debugging information.') + zipdata = to_text(zipdata, errors='surrogate_or_strict') shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars) if shebang is None: @@ -674,7 +687,7 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta shebang=shebang, interpreter=interpreter, coding=ENCODING_STRING, - ))) + ))) module_data = output.getvalue() elif module_substyle == 'powershell': @@ -721,12 +734,11 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # The main event -- substitute the JSON args string into the module module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) - facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict') + facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict') module_data = module_data.replace(b'syslog.LOG_USER', facility) return (module_data, module_style, shebang) -# ****************************************************************************** def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'): """ @@ -760,7 +772,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) if module_style == 'binary': - return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) + return (module_data, module_style, to_text(shebang, nonstring='passthru')) elif shebang is None: lines = module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): @@ -769,7 +781,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul interpreter = args[0] interpreter = to_bytes(interpreter) - new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru') + new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='surrogate_or_strict', nonstring='passthru') if new_shebang: lines[0] = shebang = new_shebang @@ -781,6 +793,6 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul module_data = b"\n".join(lines) else: - shebang = to_bytes(shebang, errors='strict') + shebang = to_bytes(shebang, errors='surrogate_or_strict') - return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) + return (module_data, module_style, to_text(shebang, nonstring='passthru')) diff --git a/lib/ansible/executor/playbook_executor.py b/lib/ansible/executor/playbook_executor.py index 3d6d54cbe2..058230d844 100644 --- a/lib/ansible/executor/playbook_executor.py +++ b/lib/ansible/executor/playbook_executor.py @@ -21,15 +21,13 @@ __metaclass__ = type import os -from ansible.compat.six import string_types - from ansible import constants as C from ansible.executor.task_queue_manager import TaskQueueManager +from ansible.module_utils._text import to_native, to_text from ansible.playbook import Playbook from ansible.template import Templar from ansible.utils.helpers import pct_to_int from ansible.utils.path import makedirs_safe -from ansible.utils.unicode import to_unicode, to_str try: from __main__ import display @@ -74,7 +72,7 @@ class PlaybookExecutor: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path))) - if self._tqm is None: # we are doing a listing + if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] else: @@ -84,7 +82,7 @@ class PlaybookExecutor: i = 1 plays = pb.get_plays() - display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path))) + display.vv(u'%d plays in %s' % (len(plays), to_text(playbook_path))) for play in plays: if play._included_path is not None: @@ -110,7 +108,7 @@ class PlaybookExecutor: if self._tqm: self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default) - else: # we are either in --list-