From 6f6bdf79140b8cdfed1004d055d9b87d0c65bdfc Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Thu, 18 May 2017 19:41:00 +0200 Subject: [PATCH] ansible/parsing: PEP8 compliancy (#24701) - Make PEP8 compliant --- lib/ansible/parsing/dataloader.py | 12 +++++---- lib/ansible/parsing/mod_args.py | 22 ++++++----------- lib/ansible/parsing/quoting.py | 1 + lib/ansible/parsing/splitter.py | 16 ++++++------ lib/ansible/parsing/utils/__init__.py | 1 - lib/ansible/parsing/utils/addresses.py | 7 +++--- lib/ansible/parsing/utils/jsonify.py | 2 +- lib/ansible/parsing/vault/__init__.py | 33 +++++++++++++------------ lib/ansible/parsing/yaml/__init__.py | 1 - lib/ansible/parsing/yaml/constructor.py | 18 +++++++------- lib/ansible/parsing/yaml/dumper.py | 5 ++-- test/sanity/pep8/legacy-files.txt | 11 --------- 12 files changed, 58 insertions(+), 71 deletions(-) diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py index db94c47449..f531faccf8 100644 --- a/lib/ansible/parsing/dataloader.py +++ b/lib/ansible/parsing/dataloader.py @@ -237,15 +237,17 @@ class DataLoader: b_main = b'main%s' % (suffix) b_tasked = b'tasks/%s' % (b_main) - if b_path.endswith(b'tasks') and os.path.exists(os.path.join(b_path, b_main)) \ - or os.path.exists(os.path.join(b_upath, b_tasked)) \ - or os.path.exists(os.path.join(os.path.dirname(b_path), b_tasked)): + if ( + b_path.endswith(b'tasks') and + os.path.exists(os.path.join(b_path, b_main)) or + os.path.exists(os.path.join(b_upath, b_tasked)) or + os.path.exists(os.path.join(os.path.dirname(b_path), b_tasked)) + ): isit = True break return isit - def path_dwim_relative(self, path, dirname, source, is_role=False): ''' find one file in either a role or playbook dir with or without @@ -283,7 +285,7 @@ class DataLoader: search.append(self.path_dwim(os.path.join(basedir, 'tasks', source))) # try to create absolute path for loader basedir + templates/files/vars + filename - search.append(self.path_dwim(os.path.join(dirname,source))) + search.append(self.path_dwim(os.path.join(dirname, source))) search.append(self.path_dwim(os.path.join(basedir, source))) # try to create absolute path for loader basedir + filename diff --git a/lib/ansible/parsing/mod_args.py b/lib/ansible/parsing/mod_args.py index c309bf1bd7..2ed4b9202c 100644 --- a/lib/ansible/parsing/mod_args.py +++ b/lib/ansible/parsing/mod_args.py @@ -19,12 +19,11 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type - -from ansible.errors import AnsibleParserError,AnsibleError +from ansible.errors import AnsibleParserError, AnsibleError from ansible.module_utils.six import iteritems, string_types from ansible.module_utils._text import to_text -from ansible.plugins import module_loader from ansible.parsing.splitter import parse_kv, split_args +from ansible.plugins import module_loader from ansible.template import Templar @@ -96,7 +95,6 @@ class ModuleArgsParser: assert isinstance(task_ds, dict) self._task_ds = task_ds - def _split_module_string(self, module_string): ''' when module names are expressed like: @@ -111,7 +109,6 @@ class ModuleArgsParser: else: return (tokens[0], "") - def _handle_shell_weirdness(self, action, args): ''' given an action name and an args dictionary, return the @@ -253,12 +250,11 @@ class ModuleArgsParser: task, dealing with all sorts of levels of fuzziness. ''' - thing = None + thing = None - action = None + action = None delegate_to = self._task_ds.get('delegate_to', None) - args = dict() - + args = dict() # This is the standard YAML form for command-type modules. We grab # the args and pass them in as additional arguments, which can/will @@ -272,7 +268,6 @@ class ModuleArgsParser: thing = self._task_ds['action'] action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) - # local_action if 'local_action' in self._task_ds: # local_action is similar but also implies a delegate_to @@ -294,14 +289,13 @@ class ModuleArgsParser: thing = value action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) - # if we didn't see any module in the task at all, it's not a task really if action is None: if 'ping' not in module_loader: raise AnsibleParserError("The requested action was not found in configured module paths. " - "Additionally, core modules are missing. If this is a checkout, " - "run 'git pull --rebase' to correct this problem.", - obj=self._task_ds) + "Additionally, core modules are missing. If this is a checkout, " + "run 'git pull --rebase' to correct this problem.", + obj=self._task_ds) else: raise AnsibleParserError("no action detected in task. This often indicates a misspelled module name, or incorrect module path.", diff --git a/lib/ansible/parsing/quoting.py b/lib/ansible/parsing/quoting.py index 7b94f90653..d3a38d946c 100644 --- a/lib/ansible/parsing/quoting.py +++ b/lib/ansible/parsing/quoting.py @@ -23,6 +23,7 @@ __metaclass__ = type def is_quoted(data): return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\' + def unquote(data): ''' removes first and last quotes from a string, if the string starts and ends with the same quotes ''' if is_quoted(data): diff --git a/lib/ansible/parsing/splitter.py b/lib/ansible/parsing/splitter.py index e6cb452035..5008dd3a6a 100644 --- a/lib/ansible/parsing/splitter.py +++ b/lib/ansible/parsing/splitter.py @@ -19,8 +19,8 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import re import codecs +import re from ansible.errors import AnsibleParserError from ansible.module_utils._text import to_text @@ -36,7 +36,7 @@ _ESCAPE_SEQUENCE_RE = re.compile(r''' | \\x{2} # 2-digit hex escapes | \\N\{{[^}}]+\}} # Unicode characters by name | \\[\\'"abfnrtv] # Single-character escapes - )'''.format(_HEXCHAR*8, _HEXCHAR*4, _HEXCHAR*2), re.UNICODE | re.VERBOSE) + )'''.format(_HEXCHAR * 8, _HEXCHAR * 4, _HEXCHAR * 2), re.UNICODE | re.VERBOSE) def _decode_escapes(s): @@ -112,7 +112,7 @@ def _get_quote_state(token, quote_char): prev_char = None for idx, cur_char in enumerate(token): if idx > 0: - prev_char = token[idx-1] + prev_char = token[idx - 1] if cur_char in '"\'' and prev_char != '\\': if quote_char: if cur_char == quote_char: @@ -128,7 +128,7 @@ def _count_jinja2_blocks(token, cur_depth, open_token, close_token): given opening/closing type and adjusts the current depth for that block based on the difference ''' - num_open = token.count(open_token) + num_open = token.count(open_token) num_close = token.count(close_token) if num_open != num_close: cur_depth += (num_open - num_close) @@ -171,13 +171,13 @@ def split_args(args): quote_char = None inside_quotes = False - print_depth = 0 # used to count nested jinja2 {{ }} blocks - block_depth = 0 # used to count nested jinja2 {% %} blocks + print_depth = 0 # used to count nested jinja2 {{ }} blocks + block_depth = 0 # used to count nested jinja2 {% %} blocks comment_depth = 0 # used to count nested jinja2 {# #} blocks # now we loop over each split chunk, coalescing tokens if the white space # split occurred within quotes or a jinja2 block of some kind - for itemidx,item in enumerate(items): + for (itemidx, item) in enumerate(items): # we split on spaces and newlines separately, so that we # can tell which character we split on for reassembly @@ -185,7 +185,7 @@ def split_args(args): tokens = item.strip().split(' ') line_continuation = False - for idx,token in enumerate(tokens): + for (idx, token) in enumerate(tokens): # if we hit a line continuation character, but # we're not inside quotes, ignore it and continue diff --git a/lib/ansible/parsing/utils/__init__.py b/lib/ansible/parsing/utils/__init__.py index 785fc45992..ae8ccff595 100644 --- a/lib/ansible/parsing/utils/__init__.py +++ b/lib/ansible/parsing/utils/__init__.py @@ -18,4 +18,3 @@ # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type - diff --git a/lib/ansible/parsing/utils/addresses.py b/lib/ansible/parsing/utils/addresses.py index ebfd850ac6..0096af441e 100644 --- a/lib/ansible/parsing/utils/addresses.py +++ b/lib/ansible/parsing/utils/addresses.py @@ -113,7 +113,7 @@ patterns = { r'''^ (?:{i4}\.){{3}}{i4} # Three parts followed by dots plus one $ - '''.format(i4=ipv4_component), re.X|re.I + '''.format(i4=ipv4_component), re.X | re.I ), # This matches an IPv6 address, but also permits range expressions. @@ -143,7 +143,7 @@ patterns = { ::(?:ffff:)?(?:{0}\.){{3}}{0}| (?:0:){{5}}ffff:(?:{0}\.){{3}}{0} $ - '''.format(ipv6_component), re.X|re.I + '''.format(ipv6_component), re.X | re.I ), # This matches a hostname or host pattern including [x:y(:z)] ranges. @@ -161,11 +161,12 @@ patterns = { {label} # We must have at least one label (?:\.{label})* # Followed by zero or more .labels $ - '''.format(label=label), re.X|re.I|re.UNICODE + '''.format(label=label), re.X | re.I | re.UNICODE ), } + def parse_address(address, allow_ranges=False): """ Takes a string and returns a (host, port) tuple. If the host is None, then diff --git a/lib/ansible/parsing/utils/jsonify.py b/lib/ansible/parsing/utils/jsonify.py index 74e015a5cf..53885f1ec0 100644 --- a/lib/ansible/parsing/utils/jsonify.py +++ b/lib/ansible/parsing/utils/jsonify.py @@ -24,6 +24,7 @@ try: except ImportError: import simplejson as json + def jsonify(result, format=False): ''' format JSON output (uncompressed or uncompressed) ''' @@ -38,4 +39,3 @@ def jsonify(result, format=False): return json.dumps(result, sort_keys=True, indent=indent, ensure_ascii=False) except UnicodeDecodeError: return json.dumps(result, sort_keys=True, indent=indent) - diff --git a/lib/ansible/parsing/vault/__init__.py b/lib/ansible/parsing/vault/__init__.py index bb863201f9..9ae68c946c 100644 --- a/lib/ansible/parsing/vault/__init__.py +++ b/lib/ansible/parsing/vault/__init__.py @@ -264,7 +264,7 @@ class VaultLib: raise AnsibleError("the cipher must be set before adding a header") header = b';'.join([b_HEADER, self.b_version, - to_bytes(self.cipher_name,'utf-8', errors='strict')]) + to_bytes(self.cipher_name, 'utf-8', errors='strict')]) b_vaulttext = [header] b_vaulttext += [b_ciphertext[i:i + 80] for i in range(0, len(b_ciphertext), 80)] b_vaulttext += [b''] @@ -319,14 +319,14 @@ class VaultEditor: file_len = os.path.getsize(tmp_path) if file_len > 0: # avoid work when file was empty - max_chunk_len = min(1024*1024*2, file_len) + max_chunk_len = min(1024 * 1024 * 2, file_len) passes = 3 - with open(tmp_path, "wb") as fh: + with open(tmp_path, "wb") as fh: for _ in range(passes): - fh.seek(0, 0) + fh.seek(0, 0) # get a random chunk of data, each pass with other length - chunk_len = random.randint(max_chunk_len//2, max_chunk_len) + chunk_len = random.randint(max_chunk_len // 2, max_chunk_len) data = os.urandom(chunk_len) for _ in range(0, file_len // chunk_len): @@ -443,7 +443,7 @@ class VaultEditor: try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: - raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) + raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename))) self.write_data(plaintext, output_file or filename, shred=False) def create_file(self, filename): @@ -470,7 +470,7 @@ class VaultEditor: try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: - raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) + raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename))) if self.vault.cipher_name not in CIPHER_WRITE_WHITELIST: # we want to get rid of files encrypted with the AES cipher @@ -486,7 +486,7 @@ class VaultEditor: try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: - raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) + raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename))) return plaintext @@ -503,7 +503,7 @@ class VaultEditor: try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: - raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) + raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename))) # This is more or less an assert, see #18247 if b_new_password is None: @@ -588,7 +588,7 @@ class VaultEditor: os.chown(dest, prev.st_uid, prev.st_gid) def _editor_shell_command(self, filename): - EDITOR = os.environ.get('EDITOR','vi') + EDITOR = os.environ.get('EDITOR', 'vi') editor = shlex.split(EDITOR) editor.append(filename) @@ -623,7 +623,7 @@ class VaultAES: b_d += b_di b_key = b_d[:key_length] - b_iv = b_d[key_length:key_length+iv_length] + b_iv = b_d[key_length:key_length + iv_length] return b_key, b_iv @@ -642,9 +642,9 @@ class VaultAES: :returns: A byte string containing the decrypted data """ - display.deprecated(u'The VaultAES format is insecure and has been' - ' deprecated since Ansible-1.5. Use vault rekey FILENAME to' - ' switch to the newer VaultAES256 format', version='2.3') + display.deprecated(u'The VaultAES format is insecure and has been ' + 'deprecated since Ansible-1.5. Use vault rekey FILENAME to ' + 'switch to the newer VaultAES256 format', version='2.3') # http://stackoverflow.com/a/14989032 b_ciphertext = unhexlify(b_vaulttext) @@ -712,10 +712,11 @@ class VaultAES256: hash_function = SHA256 # make two keys and one iv - pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest() + def pbkdf2_prf(p, s): + return HMAC.new(p, s, hash_function).digest() b_derivedkey = PBKDF2(b_password, b_salt, dkLen=(2 * keylength) + ivlength, - count=10000, prf=pbkdf2_prf) + count=10000, prf=pbkdf2_prf) return b_derivedkey @classmethod diff --git a/lib/ansible/parsing/yaml/__init__.py b/lib/ansible/parsing/yaml/__init__.py index 785fc45992..ae8ccff595 100644 --- a/lib/ansible/parsing/yaml/__init__.py +++ b/lib/ansible/parsing/yaml/__init__.py @@ -18,4 +18,3 @@ # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type - diff --git a/lib/ansible/parsing/yaml/constructor.py b/lib/ansible/parsing/yaml/constructor.py index 02beca8955..0e3a21acb9 100644 --- a/lib/ansible/parsing/yaml/constructor.py +++ b/lib/ansible/parsing/yaml/constructor.py @@ -24,10 +24,10 @@ from yaml.nodes import MappingNode from ansible.module_utils._text import to_bytes from ansible.parsing.vault import VaultLib -from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode -from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode, AnsibleVaultEncryptedUnicode from ansible.utils.unsafe_proxy import wrap_var + try: from __main__ import display except ImportError: @@ -56,8 +56,8 @@ class AnsibleConstructor(SafeConstructor): # (pyyaml silently allows overwriting keys) if not isinstance(node, MappingNode): raise ConstructorError(None, None, - "expected a mapping node, but found %s" % node.id, - node.start_mark) + "expected a mapping node, but found %s" % node.id, + node.start_mark) self.flatten_mapping(node) mapping = AnsibleMapping() @@ -70,11 +70,11 @@ class AnsibleConstructor(SafeConstructor): hash(key) except TypeError as exc: raise ConstructorError("while constructing a mapping", node.start_mark, - "found unacceptable key (%s)" % exc, key_node.start_mark) + "found unacceptable key (%s)" % exc, key_node.start_mark) if key in mapping: display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).' - u' Using last defined value only.'.format(key, *mapping.ansible_pos)) + u' Using last defined value only.'.format(key, *mapping.ansible_pos)) value = self.construct_object(value_node, deep=deep) mapping[key] = value @@ -99,8 +99,7 @@ class AnsibleConstructor(SafeConstructor): ciphertext_data = to_bytes(value) if self._b_vault_password is None: - raise ConstructorError(None, None, - "found vault but no vault password provided", node.start_mark) + raise ConstructorError(None, None, "found vault but no vault password provided", node.start_mark) # could pass in a key id here to choose the vault to associate with vault = self._vaults['default'] @@ -159,4 +158,5 @@ AnsibleConstructor.add_constructor( AnsibleConstructor.add_constructor( u'!vault', AnsibleConstructor.construct_vault_encrypted_unicode) -AnsibleConstructor.add_constructor( u'!vault-encrypted', AnsibleConstructor.construct_vault_encrypted_unicode) + +AnsibleConstructor.add_constructor(u'!vault-encrypted', AnsibleConstructor.construct_vault_encrypted_unicode) diff --git a/lib/ansible/parsing/yaml/dumper.py b/lib/ansible/parsing/yaml/dumper.py index 313169490a..8d8ffe5701 100644 --- a/lib/ansible/parsing/yaml/dumper.py +++ b/lib/ansible/parsing/yaml/dumper.py @@ -22,8 +22,7 @@ __metaclass__ = type import yaml from ansible.module_utils.six import PY3 -from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping -from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping, AnsibleVaultEncryptedUnicode from ansible.utils.unsafe_proxy import AnsibleUnsafeText from ansible.vars.hostvars import HostVars @@ -35,9 +34,11 @@ class AnsibleDumper(yaml.SafeDumper): ''' pass + def represent_hostvars(self, data): return self.represent_dict(dict(data)) + # Note: only want to represent the encrypted data def represent_vault_encrypted_unicode(self, data): return self.represent_scalar(u'!vault', data._ciphertext.decode(), style='|') diff --git a/test/sanity/pep8/legacy-files.txt b/test/sanity/pep8/legacy-files.txt index da861040dc..03d0c19acd 100644 --- a/test/sanity/pep8/legacy-files.txt +++ b/test/sanity/pep8/legacy-files.txt @@ -682,17 +682,6 @@ lib/ansible/modules/utilities/logic/async_wrapper.py lib/ansible/modules/utilities/logic/wait_for.py lib/ansible/modules/web_infrastructure/ejabberd_user.py lib/ansible/modules/web_infrastructure/jboss.py -lib/ansible/parsing/dataloader.py -lib/ansible/parsing/mod_args.py -lib/ansible/parsing/quoting.py -lib/ansible/parsing/splitter.py -lib/ansible/parsing/utils/__init__.py -lib/ansible/parsing/utils/addresses.py -lib/ansible/parsing/utils/jsonify.py -lib/ansible/parsing/vault/__init__.py -lib/ansible/parsing/yaml/__init__.py -lib/ansible/parsing/yaml/constructor.py -lib/ansible/parsing/yaml/dumper.py lib/ansible/playbook/__init__.py lib/ansible/playbook/attribute.py lib/ansible/playbook/base.py