From b4b24a0889ae52a4ee88e5754eef97ed258915a7 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 5 Feb 2016 00:39:09 -0500 Subject: [PATCH] moved to base class for shell plugins fish now sets env vars correctly fish checksum now works fixed and cleaned up imports fixed typo --- lib/ansible/plugins/shell/__init__.py | 96 +++++++++++++++++++++++-- lib/ansible/plugins/shell/csh.py | 8 ++- lib/ansible/plugins/shell/fish.py | 59 ++++++++++++++- lib/ansible/plugins/shell/powershell.py | 2 - lib/ansible/plugins/shell/sh.py | 85 +--------------------- 5 files changed, 158 insertions(+), 92 deletions(-) diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py index 785fc45992..50530c6b69 100644 --- a/lib/ansible/plugins/shell/__init__.py +++ b/lib/ansible/plugins/shell/__init__.py @@ -1,6 +1,6 @@ -# (c) 2012-2014, Michael DeHaan +# (c) 2016 RedHat # -# This file is part of Ansible +# This file is part of Ansible. # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,8 +14,96 @@ # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . - -# Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import os +import re +import pipes +import ansible.constants as C +import time +import random + +from ansible.compat.six import text_type + +_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$') + +class ShellBase(object): + + def __init__(self): + self.env = dict( + LANG = C.DEFAULT_MODULE_LANG, + LC_ALL = C.DEFAULT_MODULE_LANG, + LC_MESSAGES = C.DEFAULT_MODULE_LANG, + ) + + def env_prefix(self, **kwargs): + env = self.env.copy() + env.update(kwargs) + return ' '.join(['%s=%s' % (k, pipes.quote(text_type(v))) for k,v in env.items()]) + + def join_path(self, *args): + return os.path.join(*args) + + # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say + def get_remote_filename(self, base_name): + return base_name.strip() + + def path_has_trailing_slash(self, path): + return path.endswith('/') + + def chmod(self, mode, path): + path = pipes.quote(path) + return 'chmod %s %s' % (mode, path) + + def remove(self, path, recurse=False): + path = pipes.quote(path) + cmd = 'rm -f ' + if recurse: + cmd += '-r ' + return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL) + + def mkdtemp(self, basefile=None, system=False, mode=None): + if not basefile: + basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48)) + basetmp = self.join_path(C.DEFAULT_REMOTE_TMP, basefile) + if system and (basetmp.startswith('$HOME') or basetmp.startswith('~/')): + basetmp = self.join_path('/tmp', basefile) + cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) + cmd += ' %s echo %s echo %s %s' % (self._SHELL_AND, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) + + # change the umask in a subshell to achieve the desired mode + # also for directories created with `mkdir -p` + if mode: + tmp_umask = 0o777 & ~mode + cmd = '%s umask %o %s %s %s' % (self._SHELL_GROUP_LEFT, tmp_umask, self._SHELL_AND, cmd, self._SHELL_GROUP_RIGHT) + + return cmd + + def expand_user(self, user_home_path): + ''' Return a command to expand tildes in a path + + It can be either "~" or "~username". We use the POSIX definition of + a username: + http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426 + http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276 + ''' + + # Check that the user_path to expand is safe + if user_home_path != '~': + if not _USER_HOME_PATH_RE.match(user_home_path): + # pipes.quote will make the shell return the string verbatim + user_home_path = pipes.quote(user_home_path) + return 'echo %s' % user_home_path + + def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None): + # don't quote the cmd if it's an empty string, because this will break pipelining mode + if cmd.strip() != '': + cmd = pipes.quote(cmd) + cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd] + if arg_path is not None: + cmd_parts.append(arg_path) + new_cmd = " ".join(cmd_parts) + if rm_tmp: + new_cmd = '%s; rm -rf "%s" %s' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL) + return new_cmd diff --git a/lib/ansible/plugins/shell/csh.py b/lib/ansible/plugins/shell/csh.py index 6f1008be01..c4d6319dc5 100644 --- a/lib/ansible/plugins/shell/csh.py +++ b/lib/ansible/plugins/shell/csh.py @@ -17,9 +17,9 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.plugins.shell.sh import ShellModule as ShModule +from ansible.plugins.shell import ShellBase -class ShellModule(ShModule): +class ShellModule(ShellBase): # Common shell filenames that this plugin handles COMPATIBLE_SHELLS = frozenset(('csh', 'tcsh')) @@ -29,8 +29,12 @@ class ShellModule(ShModule): # How to end lines in a python script one-liner _SHELL_EMBEDDED_PY_EOL = '\\\n' _SHELL_REDIRECT_ALLNULL = '>& /dev/null' + _SHELL_AND = '&&' + _SHELL_OR = '||' _SHELL_SUB_LEFT = '"`' _SHELL_SUB_RIGHT = '`"' + _SHELL_GROUP_LEFT = '(' + _SHELL_GROUP_RIGHT = ')' def env_prefix(self, **kwargs): return 'env %s' % super(ShellModule, self).env_prefix(**kwargs) diff --git a/lib/ansible/plugins/shell/fish.py b/lib/ansible/plugins/shell/fish.py index aee4cf0867..ddee24ac6d 100644 --- a/lib/ansible/plugins/shell/fish.py +++ b/lib/ansible/plugins/shell/fish.py @@ -17,7 +17,9 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import pipes from ansible.plugins.shell.sh import ShellModule as ShModule +from ansible.compat.six import text_type class ShellModule(ShModule): @@ -26,6 +28,8 @@ class ShellModule(ShModule): # Family of shells this has. Must match the filename without extension SHELL_FAMILY = 'fish' + _SHELL_EMBEDDED_PY_EOL = '\n' + _SHELL_REDIRECT_ALLNULL = '> /dev/null 2>&1' _SHELL_AND = '; and' _SHELL_OR = '; or' _SHELL_SUB_LEFT = '(' @@ -34,4 +38,57 @@ class ShellModule(ShModule): _SHELL_GROUP_RIGHT = '' def env_prefix(self, **kwargs): - return 'env %s' % super(ShellModule, self).env_prefix(**kwargs) + env = self.env.copy() + env.update(kwargs) + return ' '.join(['set -lx %s %s;' % (k, pipes.quote(text_type(v))) for k,v in env.items()]) + + def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None): + # don't quote the cmd if it's an empty string, because this will break pipelining mode + if cmd.strip() != '': + cmd = pipes.quote(cmd) + cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd] + if arg_path is not None: + cmd_parts.append(arg_path) + new_cmd = " ".join(cmd_parts) + if rm_tmp: + new_cmd = 'begin ; %s; rm -rf "%s" %s ; end' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL) + return new_cmd + + def checksum(self, path, python_interp): + # The following test is fish-compliant. + # + # In the following test, each condition is a check and logical + # comparison (or or and) that sets the rc value. Every check is run so + # the last check in the series to fail will be the rc that is + # returned. + # + # If a check fails we error before invoking the hash functions because + # hash functions may successfully take the hash of a directory on BSDs + # (UFS filesystem?) which is not what the rest of the ansible code + # expects + # + # If all of the available hashing methods fail we fail with an rc of + # 0. This logic is added to the end of the cmd at the bottom of this + # function. + + # Return codes: + # checksum: success! + # 0: Unknown error + # 1: Remote file does not exist + # 2: No read permissions on the file + # 3: File is a directory + # 4: No python interpreter + + # Quoting gets complex here. We're writing a python string that's + # used by a variety of shells on the remote host to invoke a python + # "one-liner". + shell_escaped_path = pipes.quote(path) + test = "set rc flag; [ -r %(p)s ] %(shell_or)s set rc 2; [ -f %(p)s ] %(shell_or)s set rc 1; [ -d %(p)s ] %(shell_and)s set rc 3; %(i)s -V 2>/dev/null %(shell_or)s set rc 4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"$rc \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR) + csums = [ + u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python > 2.4 (including python3) + u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python == 2.4 + ] + + cmd = (" %s " % self._SHELL_OR).join(csums) + cmd = "%s; %s %s (echo \'0 \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path) + return cmd diff --git a/lib/ansible/plugins/shell/powershell.py b/lib/ansible/plugins/shell/powershell.py index acde565e2f..72f2570549 100644 --- a/lib/ansible/plugins/shell/powershell.py +++ b/lib/ansible/plugins/shell/powershell.py @@ -20,9 +20,7 @@ __metaclass__ = type import base64 import os import re -import random import shlex -import time from ansible.utils.unicode import to_bytes, to_unicode diff --git a/lib/ansible/plugins/shell/sh.py b/lib/ansible/plugins/shell/sh.py index 1e69665c0f..671eb7139e 100644 --- a/lib/ansible/plugins/shell/sh.py +++ b/lib/ansible/plugins/shell/sh.py @@ -17,18 +17,12 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import os -import re import pipes -import ansible.constants as C -import time -import random -from ansible.compat.six import text_type +from ansible.plugins.shell import ShellBase -_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$') -class ShellModule(object): +class ShellModule(ShellBase): # Common shell filenames that this plugin handles. # Note: sh is the default shell plugin so this plugin may also be selected @@ -47,69 +41,6 @@ class ShellModule(object): _SHELL_GROUP_LEFT = '(' _SHELL_GROUP_RIGHT = ')' - def env_prefix(self, **kwargs): - '''Build command prefix with environment variables.''' - env = dict( - LANG = C.DEFAULT_MODULE_LANG, - LC_ALL = C.DEFAULT_MODULE_LANG, - LC_MESSAGES = C.DEFAULT_MODULE_LANG, - ) - env.update(kwargs) - return ' '.join(['%s=%s' % (k, pipes.quote(text_type(v))) for k,v in env.items()]) - - def join_path(self, *args): - return os.path.join(*args) - - # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say - def get_remote_filename(self, base_name): - return base_name.strip() - - def path_has_trailing_slash(self, path): - return path.endswith('/') - - def chmod(self, mode, path): - path = pipes.quote(path) - return 'chmod %s %s' % (mode, path) - - def remove(self, path, recurse=False): - path = pipes.quote(path) - cmd = 'rm -f ' - if recurse: - cmd += '-r ' - return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL) - - def mkdtemp(self, basefile=None, system=False, mode=None): - if not basefile: - basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48)) - basetmp = self.join_path(C.DEFAULT_REMOTE_TMP, basefile) - if system and (basetmp.startswith('$HOME') or basetmp.startswith('~/')): - basetmp = self.join_path('/tmp', basefile) - cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) - cmd += ' %s echo %s echo %s %s' % (self._SHELL_AND, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) - - # change the umask in a subshell to achieve the desired mode - # also for directories created with `mkdir -p` - if mode: - tmp_umask = 0o777 & ~mode - cmd = '%s umask %o %s %s %s' % (self._SHELL_GROUP_LEFT, tmp_umask, self._SHELL_AND, cmd, self._SHELL_GROUP_RIGHT) - - return cmd - - def expand_user(self, user_home_path): - ''' Return a command to expand tildes in a path - - It can be either "~" or "~username". We use the POSIX definition of - a username: - http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426 - http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276 - ''' - - # Check that the user_path to expand is safe - if user_home_path != '~': - if not _USER_HOME_PATH_RE.match(user_home_path): - # pipes.quote will make the shell return the string verbatim - user_home_path = pipes.quote(user_home_path) - return 'echo %s' % user_home_path def checksum(self, path, python_interp): # The following test needs to be SH-compliant. BASH-isms will @@ -151,15 +82,3 @@ class ShellModule(object): cmd = "%s; %s %s (echo \'0 \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path) return cmd - def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None): - # don't quote the cmd if it's an empty string, because this will - # break pipelining mode - if cmd.strip() != '': - cmd = pipes.quote(cmd) - cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd] - if arg_path is not None: - cmd_parts.append(arg_path) - new_cmd = " ".join(cmd_parts) - if rm_tmp: - new_cmd = '%s; rm -rf "%s" %s' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL) - return new_cmd