Fix paramiko's exec_command() to return bytes on python3 (#17372)

* Fix paramiko's exec_command() to return bytes on python3

* Run test_connection for python3 now too

* Fix atomic_move for problem in shippable's testing

* Python-2.4 needs to use b()

lib/ansible/module_utils/basic.py

@@ -1947,12 +1947,12 @@ class AnsibleModule(object):
                 self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e))
             else:
                 b_dest_dir = os.path.dirname(b_dest)
-                # Converting from bytes so that if py3, it will be
-                # surrogateescaped.  If py2, it wil be a noop.  Converting
-                # from text strings could mangle filenames on py2)
-                native_dest_dir = to_native(b_dest_dir)
-                native_suffix = to_native(os.path.basename(b_dest))
-                native_prefix = '.ansible_tmp'
+                # Use bytes here.  In the shippable CI, this fails with
+                # a UnicodeError with surrogateescape'd strings for an unknown
+                # reason (doesn't happen in a local Ubuntu16.04 VM)
+                native_dest_dir = b_dest_dir
+                native_suffix = os.path.basename(b_dest)
+                native_prefix = b('.ansible_tmp')
                 try:
                     tmp_dest_fd, tmp_dest_name = tempfile.mkstemp(
                         prefix=native_prefix, dir=native_dest_dir, suffix=native_suffix)

lib/ansible/plugins/connection/paramiko_ssh.py

@@ -40,6 +40,7 @@ from binascii import hexlify
 from ansible.compat.six import iteritems
 
 from ansible import constants as C
+from ansible.compat.six.moves import input
 from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
 from ansible.plugins.connection import ConnectionBase
 from ansible.utils.path import makedirs_safe
@@ -100,7 +101,7 @@ class MyAddPolicy(object):
             fingerprint = hexlify(key.get_fingerprint())
             ktype = key.get_name()
 
-            inp = raw_input(AUTHENTICITY_MSG % (hostname, ktype, fingerprint))
+            inp = input(AUTHENTICITY_MSG % (hostname, ktype, fingerprint))
             sys.stdin = old_stdin
 
             self.connection.connection_unlock()
@@ -274,9 +275,9 @@ class Connection(ConnectionBase):
 
         cmd = to_bytes(cmd, errors='strict')
 
-        no_prompt_out = ''
-        no_prompt_err = ''
-        become_output = ''
+        no_prompt_out = b''
+        no_prompt_err = b''
+        become_output = b''
 
         try:
             chan.exec_command(cmd)
@@ -317,8 +318,8 @@ class Connection(ConnectionBase):
         except socket.timeout:
             raise AnsibleError('ssh timed out waiting for privilege escalation.\n' + become_output)
 
-        stdout = ''.join(chan.makefile('rb', bufsize))
-        stderr = ''.join(chan.makefile_stderr('rb', bufsize))
+        stdout = b''.join(chan.makefile('rb', bufsize))
+        stderr = b''.join(chan.makefile_stderr('rb', bufsize))
 
         return (chan.recv_exit_status(), no_prompt_out + stdout, no_prompt_out + stderr)
 

test/utils/shippable/python3-test-target-blacklist.txt

@@ -2,4 +2,3 @@ s/ pull / /
 s/ no_log / /
 s/ test_async_conditional / /
 s/ test_binary_modules$/ /
-s/ test_connection / /