More fixes for unicode handling in the connection plugins.

Tested that ssh, docker, local, lxc-libvirt, chroot all work with the
updated unicode integration test.

lib/ansible/inventory/script.py

@@ -31,6 +31,7 @@ from ansible.errors import AnsibleError
 from ansible.inventory.host import Host
 from ansible.inventory.group import Group
 from ansible.module_utils.basic import json_dict_bytes_to_unicode
+from ansible.utils.unicode import to_str
 
 
 class InventoryScript:
@@ -72,11 +73,11 @@ class InventoryScript:
             self.raw = self._loader.load(self.data)
         except Exception as e:
             sys.stderr.write(err + "\n")
-            raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(self.filename, str(e)))
+            raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename_, to_str(e)))
 
         if not isinstance(self.raw, Mapping):
             sys.stderr.write(err + "\n")
-            raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(self.filename))
+            raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename)))
 
         self.raw  = json_dict_bytes_to_unicode(self.raw)
 

lib/ansible/plugins/connection/docker.py

@@ -113,7 +113,7 @@ class Connection(ConnectionBase):
         """ Connect to the container. Nothing to do """
         super(Connection, self)._connect()
         if not self._connected:
-            display.vvv("ESTABLISH DOCKER CONNECTION FOR USER: {0}".format(
+            display.vvv(u"ESTABLISH DOCKER CONNECTION FOR USER: {0}".format(
                 self._play_context.remote_user, host=self._play_context.remote_addr)
             )
             self._connected = True
@@ -172,7 +172,7 @@ class Connection(ConnectionBase):
             # running containers, so we use docker exec to implement this
             executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh'
             args = [self.docker_cmd, "exec", "-i", self._play_context.remote_addr, executable, "-c",
-                    "dd of={0} bs={1}".format(out_path, BUFSIZE)]
+                    "dd of=%s bs=%s" % (out_path, BUFSIZE)]
             args = map(to_bytes, args)
             with open(in_path, 'rb') as in_file:
                 try:

lib/ansible/plugins/connection/local.py

@@ -31,7 +31,7 @@ import ansible.constants as C
 
 from ansible.errors import AnsibleError, AnsibleFileNotFound
 from ansible.plugins.connection import ConnectionBase
-from ansible.utils.unicode import to_bytes
+from ansible.utils.unicode import to_bytes, to_str
 
 try:
     from __main__ import display
@@ -57,7 +57,7 @@ class Connection(ConnectionBase):
         self._play_context.remote_user = getpass.getuser()
 
         if not self._connected:
-            display.vvv("ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user, host=self._play_context.remote_addr))
+            display.vvv(u"ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user, host=self._play_context.remote_addr))
             self._connected = True
         return self
 
@@ -126,22 +126,22 @@ class Connection(ConnectionBase):
 
         super(Connection, self).put_file(in_path, out_path)
 
-        display.vvv("{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
+        display.vvv(u"{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
         if not os.path.exists(in_path):
-            raise AnsibleFileNotFound("file or module does not exist: {0}".format(in_path))
+            raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
         try:
             shutil.copyfile(in_path, out_path)
         except shutil.Error:
-            raise AnsibleError("failed to copy: {0} and {1} are the same".format(in_path, out_path))
+            raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path)))
         except IOError as e:
-            raise AnsibleError("failed to transfer file to {0}: {1}".format(out_path, e))
+            raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e)))
 
     def fetch_file(self, in_path, out_path):
         ''' fetch a file from local to local -- for copatibility '''
 
         super(Connection, self).fetch_file(in_path, out_path)
 
-        display.vvv("{0} FETCH {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
+        display.vvv(u"{0} FETCH {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
         self.put_file(in_path, out_path)
 
     def close(self):

lib/ansible/plugins/connection/ssh.py

@@ -32,7 +32,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
 from ansible.plugins.connection import ConnectionBase
 from ansible.utils.path import unfrackpath, makedirs_safe
-from ansible.utils.unicode import to_bytes, to_unicode
+from ansible.utils.unicode import to_bytes, to_unicode, to_str
 from ansible.compat.six import text_type, binary_type
 
 try:
@@ -197,7 +197,7 @@ class Connection(ConnectionBase):
         if user:
             self._add_args(
                 "ANSIBLE_REMOTE_USER/remote_user/ansible_user/user/-u set",
-                ("-o", "User={0}".format(self._play_context.remote_user))
+                ("-o", "User={0}".format(to_bytes(self._play_context.remote_user)))
             )
 
         self._add_args(
@@ -231,7 +231,7 @@ class Connection(ConnectionBase):
                     raise AnsibleError("Cannot write to ControlPath %s" % cpdir)
 
                 args = ("-o", "ControlPath={0}".format(
-                    C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir))
+                    to_bytes(C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)))
                 )
                 self._add_args("found only ControlPersist; added ControlPath", args)
 
@@ -320,7 +320,7 @@ class Connection(ConnectionBase):
         Starts the command and communicates with it until it ends.
         '''
 
-        display_cmd = map(pipes.quote, cmd)
+        display_cmd = map(to_unicode, map(pipes.quote, cmd))
         display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)
 
         # Start the given command. If we don't need to pipeline data, we can try
@@ -354,7 +354,7 @@ class Connection(ConnectionBase):
 
         if self._play_context.password:
             os.close(self.sshpass_pipe[0])
-            os.write(self.sshpass_pipe[1], "{0}\n".format(self._play_context.password))
+            os.write(self.sshpass_pipe[1], "{0}\n".format(to_bytes(self._play_context.password)))
             os.close(self.sshpass_pipe[1])
 
         ## SSH state machine
@@ -562,7 +562,7 @@ class Connection(ConnectionBase):
 
         super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
 
-        display.vvv("ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)
+        display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)
 
         # we can only use tty when we are not pipelining the modules. piping
         # data into /usr/bin/python inside a tty automatically invokes the
@@ -630,44 +630,46 @@ class Connection(ConnectionBase):
 
         super(Connection, self).put_file(in_path, out_path)
 
-        display.vvv("PUT {0} TO {1}".format(in_path, out_path), host=self.host)
+        display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
         if not os.path.exists(in_path):
-            raise AnsibleFileNotFound("file or module does not exist: {0}".format(in_path))
+            raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
 
         # scp and sftp require square brackets for IPv6 addresses, but
         # accept them for hostnames and IPv4 addresses too.
         host = '[%s]' % self.host
 
         if C.DEFAULT_SCP_IF_SSH:
-            cmd = self._build_command('scp', in_path, '{0}:{1}'.format(host, pipes.quote(out_path)))
+            cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, pipes.quote(out_path)))
             in_data = None
         else:
-            cmd = self._build_command('sftp', host)
+            cmd = self._build_command('sftp', to_bytes(host))
-            in_data = "put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
+            in_data = u"put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
 
+        in_data = to_bytes(in_data, nonstring='passthru')
         (returncode, stdout, stderr) = self._run(cmd, in_data)
 
         if returncode != 0:
-            raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(out_path, stdout, stderr))
+            raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr)))
 
     def fetch_file(self, in_path, out_path):
         ''' fetch a file from remote to local '''
 
         super(Connection, self).fetch_file(in_path, out_path)
 
-        display.vvv("FETCH {0} TO {1}".format(in_path, out_path), host=self.host)
+        display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host)
 
         # scp and sftp require square brackets for IPv6 addresses, but
         # accept them for hostnames and IPv4 addresses too.
         host = '[%s]' % self.host
 
         if C.DEFAULT_SCP_IF_SSH:
-            cmd = self._build_command('scp', '{0}:{1}'.format(host, pipes.quote(in_path)), out_path)
+            cmd = self._build_command('scp', u'{0}:{1}'.format(host, pipes.quote(in_path)), out_path)
             in_data = None
         else:
             cmd = self._build_command('sftp', host)
-            in_data = "get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
+            in_data = u"get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
 
+        in_data = to_bytes(in_data, nonstring='passthru')
         (returncode, stdout, stderr) = self._run(cmd, in_data)
 
         if returncode != 0:

lib/ansible/plugins/connection/winrm.py

@@ -318,7 +318,7 @@ class Connection(ConnectionBase):
         local_sha1 = secure_hash(in_path)
 
         if not remote_sha1 == local_sha1:
-            raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(remote_sha1, local_sha1))
+            raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1)))
 
 
     def fetch_file(self, in_path, out_path):

lib/ansible/plugins/shell/sh.py

@@ -136,8 +136,8 @@ class ShellModule(object):
         shell_escaped_path = pipes.quote(path)
         test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc}  \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR)
         csums = [
-            "({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python > 2.4 (including python3)
+            u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python > 2.4 (including python3)
-            "({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python == 2.4
+            u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL),      # Python == 2.4
         ]
 
         cmd = (" %s " % self._SHELL_OR).join(csums)

test/integration/unicode.yml

@@ -93,6 +93,36 @@
         that:
           - "'Zażółć' in results.stdout_lines"
 
+    - name: Clean a temp directory
+      file:
+        path: /var/tmp/ansible_test_unicode_get_put
+        state: absent
+
+    - name: Create a temp directory
+      file:
+        path: /var/tmp/ansible_test_unicode_get_put
+        state: directory
+
+    - name: Create a file with a non-ascii filename
+      file:
+        path: /var/tmp/ansible_test_unicode_get_put/Zażółć
+        state: touch
+      delegate_to: localhost
+
+    - name: Put with unicode filename
+      copy:
+        src: /var/tmp/ansible_test_unicode_get_put/Zażółć
+        dest: /var/tmp/ansible_test_unicode_get_put/Zażółć2
+
+    - name: Fetch with unicode filename
+      fetch:
+        src: /var/tmp/ansible_test_unicode_get_put/Zażółć2
+        dest: /var/tmp/ansible_test_unicode_get_put/
+
+    - name: Clean a temp directory
+      file:
+        path: /var/tmp/ansible_test_unicode_get_put
+        state: absent
 
 - name: 'A play for hosts in group: ĪīĬĭ'
   hosts: 'ĪīĬĭ'