Enable most unittests on python3 (just some vault unittests and a logging one left) (#17240)

Make some python3 fixes to make the unittests pass:

* galaxy imports
* dictionary iteration in role requirements
* swap_stdout helper for unittests
* Normalize to text string in a facts.py function

lib/ansible/cli/galaxy.py

@@ -39,7 +39,7 @@ from ansible.galaxy.role import GalaxyRole
 from ansible.galaxy.login import GalaxyLogin
 from ansible.galaxy.token import GalaxyToken
 from ansible.playbook.role.requirement import RoleRequirement
-from ansible.utils.unicode import to_unicode
+from ansible.utils.unicode import to_bytes, to_unicode
 
 try:
     from __main__ import display
@@ -193,7 +193,7 @@ class GalaxyCLI(CLI):
             os.makedirs(role_path)
         readme_path = os.path.join(role_path, "README.md")
         f = open(readme_path, "wb")
-        f.write(self.galaxy.default_readme)
+        f.write(to_bytes(self.galaxy.default_readme))
         f.close()
 
         # create default .travis.yml

lib/ansible/galaxy/api.py

@@ -25,12 +25,11 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import json
-import urllib
-
-from urllib2 import quote as urlquote, HTTPError
 
 import ansible.constants as C
 from ansible.compat.six import string_types
+from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode
+from ansible.compat.six.moves.urllib.error import HTTPError
 from ansible.errors import AnsibleError
 from ansible.module_utils.urls import open_url
 from ansible.galaxy.token import GalaxyToken
@@ -134,7 +133,7 @@ class GalaxyAPI(object):
         Retrieve an authentication token
         """
         url = '%s/tokens/' % self.baseurl
-        args = urllib.urlencode({"github_token": github_token})
+        args = urlencode({"github_token": github_token})
         resp = open_url(url, data=args, validate_certs=self._validate_certs, method="POST")
         data = json.load(resp)
         return data
@@ -145,7 +144,7 @@ class GalaxyAPI(object):
         Post an import request
         """
         url = '%s/imports/' % self.baseurl
-        args = urllib.urlencode({
+        args = urlencode({
             "github_user": github_user,
             "github_repo": github_repo,
             "github_reference": reference if reference else ""
@@ -271,7 +270,7 @@ class GalaxyAPI(object):
     @g_connect
     def add_secret(self, source, github_user, github_repo, secret):
         url = "%s/notification_secrets/" % self.baseurl
-        args = urllib.urlencode({
+        args = urlencode({
             "source": source,
             "github_user": github_user,
             "github_repo": github_repo,

lib/ansible/galaxy/login.py

@@ -26,10 +26,9 @@ __metaclass__ = type
 
 import getpass
 import json
-import urllib
 
-from urllib2 import quote as urlquote, HTTPError
-from urlparse import urlparse
+from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlparse
+from ansible.compat.six.moves.urllib.error import HTTPError
 
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.module_utils.urls import open_url

lib/ansible/module_utils/facts.py

@@ -34,6 +34,7 @@ import pwd
 
 from ansible.module_utils.basic import get_all_subclasses
 from ansible.module_utils.six import PY3, iteritems
+from ansible.module_utils._text import to_text
 
 # py2 vs py3; replace with six via ansiballz
 try:
@@ -357,6 +358,7 @@ class Facts(object):
             proc_1 = os.path.basename(proc_1)
 
         if proc_1 is not None:
+            proc_1 = to_text(proc_1)
             proc_1 = proc_1.strip()
 
         if proc_1 == 'init' or proc_1.endswith('sh'):

lib/ansible/playbook/role/requirement.py

@@ -169,7 +169,7 @@ class RoleRequirement(RoleDefinition):
             if 'scm' not in role:
                 role['scm'] = None
 
-        for key in role.keys():
+        for key in list(role.keys()):
             if key not in VALID_SPEC_KEYS:
                 role.pop(key)
 

test/units/cli/test_galaxy.py

@@ -25,7 +25,6 @@ import tarfile
 import tempfile
 
 from mock import patch
-from nose.plugins.skip import SkipTest
 
 from ansible.compat.six import PY3
 from ansible.compat.tests import unittest
@@ -34,13 +33,9 @@ from mock import patch, call
 import ansible
 from ansible.errors import AnsibleError, AnsibleOptionsError
 
-from nose.plugins.skip import SkipTest
-
-if PY3:
-    raise SkipTest('galaxy is not ported to be py3 compatible yet')
-
 from ansible.cli.galaxy import GalaxyCLI
 
+
 class TestGalaxy(unittest.TestCase):
     @classmethod
     def setUpClass(cls):

test/units/mock/procenv.py

@@ -54,6 +54,9 @@ def swap_stdout():
     context manager that temporarily replaces stdout for tests that need to verify output
     """
     old_stdout = sys.stdout
+    if PY3:
+        fake_stream = StringIO()
+    else:
         fake_stream = BytesIO()
     sys.stdout = fake_stream
     yield fake_stream

test/units/module_utils/basic/test_exit_json.py

@@ -32,7 +32,6 @@ from ansible.module_utils import basic
 
 empty_invocation = {u'module_args': {}}
 
-@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
 class TestAnsibleModuleExitJson(unittest.TestCase):
     def setUp(self):
         args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))
@@ -90,7 +89,6 @@ class TestAnsibleModuleExitJson(unittest.TestCase):
         return_val = json.loads(self.fake_stream.getvalue())
         self.assertEquals(return_val, dict(changed=True, msg='success', invocation=empty_invocation))
 
-@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
 class TestAnsibleModuleExitValuesRemoved(unittest.TestCase):
     OMIT = 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
     dataset = (

test/units/module_utils/basic/test_run_command.py

@@ -42,7 +42,6 @@ class OpenBytesIO(BytesIO):
         pass
 
 
-@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
 class TestAnsibleModuleRunCommand(unittest.TestCase):
     def setUp(self):
         self.cmd_out = {

test/units/module_utils/test_distribution_version.py

@@ -30,6 +30,7 @@ from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch
 
 # the module we are actually testing
+import ansible.module_utils.facts as facts
 
 
 # to generate the testcase data, you can use the script gen_distribution_version_testcase.py in hacking/tests
@@ -621,7 +622,7 @@ DISTRIB_DESCRIPTION="CoreOS 976.0.0 (Coeur Rouge)"
 
 ]
 
-@unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)")
+
 def test_distribution_version():
     """tests the distribution parsing code of the Facts class
 
@@ -634,9 +635,6 @@ def test_distribution_version():
     * results for the ansible variables distribution*
     """
 
-    # needs to be in here, because the import fails with python3 still
-    import ansible.module_utils.facts as facts
-
     from ansible.module_utils import basic
 
     args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}))

test/units/parsing/vault/test_vault_editor.py

@@ -19,7 +19,6 @@
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
-#!/usr/bin/env python
 
 import sys
 import os
@@ -103,12 +102,13 @@ class TestVaultEditor(unittest.TestCase):
 
         self.assertTrue(os.path.exists(tmp_file.name))
 
+    @unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3")
     def test_decrypt_1_0(self):
         """
         Skip testing decrypting 1.0 files if we don't have access to AES, KDF or
         Counter, or we are running on python3 since VaultAES hasn't been backported.
         """
-        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2 or sys.version > '3':
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
 
         v10_file = tempfile.NamedTemporaryFile(delete=False)
@@ -163,12 +163,13 @@ class TestVaultEditor(unittest.TestCase):
         assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
 
 
+    @unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3")
     def test_rekey_migration(self):
         """
         Skip testing rekeying files if we don't have access to AES, KDF or
         Counter, or we are running on python3 since VaultAES hasn't been backported.
         """
-        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2 or sys.version > '3':
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
 
         v10_file = tempfile.NamedTemporaryFile(delete=False)