Merge pull request #14319 from bcoca/shell_plugin_revamp

moved to base class for shell plugins

lib/ansible/plugins/shell/__init__.py

@@ -1,6 +1,6 @@
-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2016 RedHat
 #
-# This file is part of Ansible
+# This file is part of Ansible.
 #
 # Ansible is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,8 +14,96 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-# Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import os
+import re
+import pipes
+import ansible.constants as C
+import time
+import random
+
+from ansible.compat.six import text_type
+
+_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$')
+
+class ShellBase(object):
+
+    def __init__(self):
+        self.env = dict(
+            LANG        = C.DEFAULT_MODULE_LANG,
+            LC_ALL      = C.DEFAULT_MODULE_LANG,
+            LC_MESSAGES = C.DEFAULT_MODULE_LANG,
+        )
+
+    def env_prefix(self, **kwargs):
+        env = self.env.copy()
+        env.update(kwargs)
+        return ' '.join(['%s=%s' % (k, pipes.quote(text_type(v))) for k,v in env.items()])
+
+    def join_path(self, *args):
+        return os.path.join(*args)
+
+    # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say
+    def get_remote_filename(self, base_name):
+        return base_name.strip()
+
+    def path_has_trailing_slash(self, path):
+        return path.endswith('/')
+
+    def chmod(self, mode, path):
+        path = pipes.quote(path)
+        return 'chmod %s %s' % (mode, path)
+
+    def remove(self, path, recurse=False):
+        path = pipes.quote(path)
+        cmd = 'rm -f '
+        if recurse:
+            cmd += '-r '
+        return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL)
+
+    def mkdtemp(self, basefile=None, system=False, mode=None):
+        if not basefile:
+            basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))
+        basetmp = self.join_path(C.DEFAULT_REMOTE_TMP, basefile)
+        if system and (basetmp.startswith('$HOME') or basetmp.startswith('~/')):
+            basetmp = self.join_path('/tmp', basefile)
+        cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT)
+        cmd += ' %s echo %s echo %s %s' % (self._SHELL_AND, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT)
+
+        # change the umask in a subshell to achieve the desired mode
+        # also for directories created with `mkdir -p`
+        if mode:
+            tmp_umask = 0o777 & ~mode
+            cmd = '%s umask %o %s %s %s' % (self._SHELL_GROUP_LEFT, tmp_umask, self._SHELL_AND, cmd, self._SHELL_GROUP_RIGHT)
+
+        return cmd
+
+    def expand_user(self, user_home_path):
+        ''' Return a command to expand tildes in a path
+
+        It can be either "~" or "~username".  We use the POSIX definition of
+        a username:
+            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426
+            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276
+        '''
+
+        # Check that the user_path to expand is safe
+        if user_home_path != '~':
+            if not _USER_HOME_PATH_RE.match(user_home_path):
+                # pipes.quote will make the shell return the string verbatim
+                user_home_path = pipes.quote(user_home_path)
+        return 'echo %s' % user_home_path
+
+    def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
+        # don't quote the cmd if it's an empty string, because this will break pipelining mode
+        if cmd.strip() != '':
+            cmd = pipes.quote(cmd)
+        cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd]
+        if arg_path is not None:
+            cmd_parts.append(arg_path)
+        new_cmd = " ".join(cmd_parts)
+        if rm_tmp:
+            new_cmd = '%s; rm -rf "%s" %s' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL)
+        return new_cmd

lib/ansible/plugins/shell/csh.py

@@ -17,9 +17,9 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible.plugins.shell.sh import ShellModule as ShModule
+from ansible.plugins.shell import ShellBase
 
-class ShellModule(ShModule):
+class ShellModule(ShellBase):
 
     # Common shell filenames that this plugin handles
     COMPATIBLE_SHELLS = frozenset(('csh', 'tcsh'))
@@ -29,8 +29,12 @@ class ShellModule(ShModule):
     # How to end lines in a python script one-liner
     _SHELL_EMBEDDED_PY_EOL = '\\\n'
     _SHELL_REDIRECT_ALLNULL = '>& /dev/null'
+    _SHELL_AND = '&&'
+    _SHELL_OR = '||'
     _SHELL_SUB_LEFT = '"`'
     _SHELL_SUB_RIGHT = '`"'
+    _SHELL_GROUP_LEFT = '('
+    _SHELL_GROUP_RIGHT = ')'
 
     def env_prefix(self, **kwargs):
         return 'env %s' % super(ShellModule, self).env_prefix(**kwargs)

lib/ansible/plugins/shell/fish.py

@@ -17,7 +17,9 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import pipes
 from ansible.plugins.shell.sh import ShellModule as ShModule
+from ansible.compat.six import text_type
 
 class ShellModule(ShModule):
 
@@ -26,6 +28,8 @@ class ShellModule(ShModule):
     # Family of shells this has.  Must match the filename without extension
     SHELL_FAMILY = 'fish'
 
+    _SHELL_EMBEDDED_PY_EOL = '\n'
+    _SHELL_REDIRECT_ALLNULL = '> /dev/null 2>&1'
     _SHELL_AND = '; and'
     _SHELL_OR = '; or'
     _SHELL_SUB_LEFT = '('
@@ -34,4 +38,57 @@ class ShellModule(ShModule):
     _SHELL_GROUP_RIGHT = ''
 
     def env_prefix(self, **kwargs):
-        return 'env %s' % super(ShellModule, self).env_prefix(**kwargs)
+        env = self.env.copy()
+        env.update(kwargs)
+        return ' '.join(['set -lx %s %s;' % (k, pipes.quote(text_type(v))) for k,v in env.items()])
+
+    def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
+        # don't quote the cmd if it's an empty string, because this will break pipelining mode
+        if cmd.strip() != '':
+            cmd = pipes.quote(cmd)
+        cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd]
+        if arg_path is not None:
+            cmd_parts.append(arg_path)
+        new_cmd = " ".join(cmd_parts)
+        if rm_tmp:
+            new_cmd = 'begin ; %s; rm -rf "%s" %s ; end' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL)
+        return new_cmd
+
+    def checksum(self, path, python_interp):
+        # The following test is fish-compliant.
+        #
+        # In the following test, each condition is a check and logical
+        # comparison (or or and) that sets the rc value.  Every check is run so
+        # the last check in the series to fail will be the rc that is
+        # returned.
+        #
+        # If a check fails we error before invoking the hash functions because
+        # hash functions may successfully take the hash of a directory on BSDs
+        # (UFS filesystem?) which is not what the rest of the ansible code
+        # expects
+        #
+        # If all of the available hashing methods fail we fail with an rc of
+        # 0.  This logic is added to the end of the cmd at the bottom of this
+        # function.
+
+        # Return codes:
+        # checksum: success!
+        # 0: Unknown error
+        # 1: Remote file does not exist
+        # 2: No read permissions on the file
+        # 3: File is a directory
+        # 4: No python interpreter
+
+        # Quoting gets complex here.  We're writing a python string that's
+        # used by a variety of shells on the remote host to invoke a python
+        # "one-liner".
+        shell_escaped_path = pipes.quote(path)
+        test = "set rc flag; [ -r %(p)s ] %(shell_or)s set rc 2; [ -f %(p)s ] %(shell_or)s set rc 1; [ -d %(p)s ] %(shell_and)s set rc 3; %(i)s -V 2>/dev/null %(shell_or)s set rc 4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"$rc  \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR)
+        csums = [
+            u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python > 2.4 (including python3)
+            u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python == 2.4
+        ]
+
+        cmd = (" %s " % self._SHELL_OR).join(csums)
+        cmd = "%s; %s %s (echo \'0  \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path)
+        return cmd

lib/ansible/plugins/shell/powershell.py

@@ -20,9 +20,7 @@ __metaclass__ = type
 import base64
 import os
 import re
-import random
 import shlex
-import time
 
 from ansible.utils.unicode import to_bytes, to_unicode
 

lib/ansible/plugins/shell/sh.py

@@ -17,18 +17,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os
-import re
 import pipes
-import ansible.constants as C
-import time
-import random
 
-from ansible.compat.six import text_type
+from ansible.plugins.shell import ShellBase
 
-_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$')
 
-class ShellModule(object):
+class ShellModule(ShellBase):
 
     # Common shell filenames that this plugin handles.
     # Note: sh is the default shell plugin so this plugin may also be selected
@@ -47,69 +41,6 @@ class ShellModule(object):
     _SHELL_GROUP_LEFT = '('
     _SHELL_GROUP_RIGHT = ')'
 
-    def env_prefix(self, **kwargs):
-        '''Build command prefix with environment variables.'''
-        env = dict(
-            LANG        = C.DEFAULT_MODULE_LANG,
-            LC_ALL      = C.DEFAULT_MODULE_LANG,
-            LC_MESSAGES = C.DEFAULT_MODULE_LANG,
-        )
-        env.update(kwargs)
-        return ' '.join(['%s=%s' % (k, pipes.quote(text_type(v))) for k,v in env.items()])
-
-    def join_path(self, *args):
-        return os.path.join(*args)
-
-    # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say
-    def get_remote_filename(self, base_name):
-        return base_name.strip()
-
-    def path_has_trailing_slash(self, path):
-        return path.endswith('/')
-
-    def chmod(self, mode, path):
-        path = pipes.quote(path)
-        return 'chmod %s %s' % (mode, path)
-
-    def remove(self, path, recurse=False):
-        path = pipes.quote(path)
-        cmd = 'rm -f '
-        if recurse:
-            cmd += '-r '
-        return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL)
-
-    def mkdtemp(self, basefile=None, system=False, mode=None):
-        if not basefile:
-            basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))
-        basetmp = self.join_path(C.DEFAULT_REMOTE_TMP, basefile)
-        if system and (basetmp.startswith('$HOME') or basetmp.startswith('~/')):
-            basetmp = self.join_path('/tmp', basefile)
-        cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT)
-        cmd += ' %s echo %s echo %s %s' % (self._SHELL_AND, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT)
-
-        # change the umask in a subshell to achieve the desired mode
-        # also for directories created with `mkdir -p`
-        if mode:
-            tmp_umask = 0o777 & ~mode
-            cmd = '%s umask %o %s %s %s' % (self._SHELL_GROUP_LEFT, tmp_umask, self._SHELL_AND, cmd, self._SHELL_GROUP_RIGHT)
-
-        return cmd
-
-    def expand_user(self, user_home_path):
-        ''' Return a command to expand tildes in a path
-
-        It can be either "~" or "~username".  We use the POSIX definition of
-        a username:
-            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426
-            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276
-        '''
-
-        # Check that the user_path to expand is safe
-        if user_home_path != '~':
-            if not _USER_HOME_PATH_RE.match(user_home_path):
-                # pipes.quote will make the shell return the string verbatim
-                user_home_path = pipes.quote(user_home_path)
-        return 'echo %s' % user_home_path
 
     def checksum(self, path, python_interp):
         # The following test needs to be SH-compliant.  BASH-isms will
@@ -151,15 +82,3 @@ class ShellModule(object):
         cmd = "%s; %s %s (echo \'0  \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path)
         return cmd
 
-    def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
-        # don't quote the cmd if it's an empty string, because this will
-        # break pipelining mode
-        if cmd.strip() != '':
-            cmd = pipes.quote(cmd)
-        cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd]
-        if arg_path is not None:
-            cmd_parts.append(arg_path)
-        new_cmd = " ".join(cmd_parts)
-        if rm_tmp:
-            new_cmd = '%s; rm -rf "%s" %s' % (new_cmd, rm_tmp, self._SHELL_REDIRECT_ALLNULL)
-        return new_cmd