Increase python version coverage for tests. (#24762)

* Improve ansible-test inventory handling.
* Fix python 3 re-raise of exception from thread.
* Fix python 3 encoding for windows-integration.
* Run network tests on multiple python versions.
* Run windows tests on multiple python versions.
* Support Shippable delegation using --tox.
* Skip vyos_command on python 3 tests until fixed.
* Add python 3 filtering to local and tox.
* Fix tests to support back to back runs.
* Temporarily test networking with python 2.7 only.

Running the tests back to back causes intermittent test failures
which need to be addressed before we can test multiple versions
in a single test run.

test/integration/inventory.networking.template

@@ -0,0 +1,14 @@
+# This is the configuration template for ansible-test network-integration tests.
+#
+# You do not need this template if you are:
+#
+# 1) Running integration tests without using ansible-test.
+# 2) Using the `--platform` option to provision temporary network instances on EC2.
+#
+# If you do not want to use the automatically provisioned temporary network instances,
+# fill in the @VAR placeholders below and save this file without the .template extension.
+#
+# NOTE: Automatic provisioning of network instances on EC2 requires an ansible-core-ci API key.
+
+[@OS]
+@NAME ansible_connection="local" ansible_host=@HOST ansible_network_os="@OS" ansible_user="@USER" ansible_ssh_private_key_file="@KEY_FILE"

test/integration/inventory.winrm.template

@@ -1,25 +1,28 @@
-[windows_hosts]
+# This is the configuration template for ansible-test windows-integration tests.
+# It can also be used with the legacy `make` based method of running tests.
+#
+# You do not need this template if you are:
+#
+# 1) Using the `--windows` option to provision temporary Windows instances on EC2.
+#
+# If you do not want to use the automatically provisioned temporary Windows instances,
+# fill in the @VAR placeholders below and save this file without the .template extension.
+#
+# NOTE: Automatic provisioning of Windows instances on EC2 requires an ansible-core-ci API key.
+#
+# REMINDER: Standard ports for winrm are 5985 (HTTP) and 5986 (HTTPS).
+
+[windows]
+@NAME ansible_host=@HOST ansible_user=@USER ansible_password=@PASSWORD ansible_port=@PORT
+
+[windows:vars]
+ansible_connection=winrm
+ansible_winrm_server_cert_validation=ignore
+
+# support winrm connection tests (temporary solution, does not support testing enable/disable of pipelining)
+[winrm:children]
 windows
-testhost_binary_modules
 
-[windows_hosts:vars]
-ansible_connection=winrm
-ansible_host=@ansible_host
-ansible_user=@ansible_user
-ansible_password=@ansible_password
-# HTTPS uses 5986, HTTP uses 5985
-ansible_port=5986
-ansible_winrm_server_cert_validation=ignore
-
-[winrm]
-winrm-pipelining    ansible_ssh_pipelining=true
-winrm-no-pipelining ansible_ssh_pipelining=false
-
-[winrm:vars]
-ansible_connection=winrm
-ansible_host=@ansible_host
-ansible_user=@ansible_user
-ansible_password=@ansible_password
-# HTTPS uses 5986, HTTP uses 5985
-ansible_port=5986
-ansible_winrm_server_cert_validation=ignore
+# support winrm binary module tests (temporary solution)
+[testhost_binary_modules:children]
+windows

test/integration/targets/vyos_command/aliases

@@ -1 +1,2 @@
 network/ci
+skip/python3

test/integration/targets/vyos_command/tests/cli/output.yaml

@@ -35,8 +35,6 @@
       - show hardware cpu detail
       - show hardware mem
       - show license
-      - show log
-      - show system boot-messages
     provider: "{{ cli }}"
   register: result
 

test/integration/targets/win_ping/aliases

@@ -1 +1,2 @@
 windows/ci/group1
+windows/ci/minimal

test/runner/lib/core_ci.py

@@ -265,8 +265,8 @@ class AnsibleCoreCI(object):
         display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1)
 
         if self.platform == 'windows':
-            with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd:
-                winrm_config = winrm_config_fd.read()
+            with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'rb') as winrm_config_fd:
+                winrm_config = winrm_config_fd.read().decode('utf-8')
         else:
             winrm_config = None
 

test/runner/lib/delegation.py

@@ -35,6 +35,8 @@ from lib.util import (
     ApplicationError,
     EnvironmentConfig,
     run_command,
+    common_environment,
+    pass_vars,
 )
 
 from lib.docker_util import (
@@ -129,7 +131,18 @@ def delegate_tox(args, exclude, require):
             if args.coverage and not args.coverage_label:
                 cmd += ['--coverage-label', 'tox-%s' % version]
 
-        run_command(args, tox + cmd)
+        env = common_environment()
+
+        # temporary solution to permit ansible-test delegated to tox to provision remote resources
+        optional = (
+            'SHIPPABLE',
+            'SHIPPABLE_BUILD_ID',
+            'SHIPPABLE_JOB_NUMBER',
+        )
+
+        env.update(pass_vars(required=[], optional=optional))
+
+        run_command(args, tox + cmd, env=env)
 
 
 def delegate_docker(args, exclude, require):

test/runner/lib/executor.py

@@ -245,6 +245,11 @@ def command_network_integration(args):
     """
     :type args: NetworkIntegrationConfig
     """
+    filename = 'test/integration/inventory.networking'
+
+    if not args.explain and not args.platform and not os.path.isfile(filename):
+        raise ApplicationError('Use the --platform option or provide an inventory file (see %s.template).' % filename)
+
     internal_targets = command_integration_filter(args, walk_network_integration_targets())
     platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
 
@@ -276,8 +281,6 @@ def command_network_integration(args):
         remotes = [instance.wait_for_result() for instance in instances]
         inventory = network_inventory(remotes)
 
-        filename = 'test/integration/inventory.networking'
-
         display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
 
         if not args.explain:
@@ -349,6 +352,11 @@ def command_windows_integration(args):
     """
     :type args: WindowsIntegrationConfig
     """
+    filename = 'test/integration/inventory.winrm'
+
+    if not args.explain and not args.windows and not os.path.isfile(filename):
+        raise ApplicationError('Use the --windows option or provide an inventory file (see %s.template).' % filename)
+
     internal_targets = command_integration_filter(args, walk_windows_integration_targets())
 
     if args.windows:
@@ -368,8 +376,6 @@ def command_windows_integration(args):
         remotes = [instance.wait_for_result() for instance in instances]
         inventory = windows_inventory(remotes)
 
-        filename = 'test/integration/inventory.winrm'
-
         display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
 
         if not args.explain:
@@ -1111,6 +1117,14 @@ def get_integration_local_filter(args, targets):
             display.warning('Excluding tests marked "%s" which require --allow-destructive to run locally: %s'
                             % (skip.rstrip('/'), ', '.join(skipped)))
 
+    if args.python_version.startswith('3'):
+        skip = 'skip/python3/'
+        skipped = [target.name for target in targets if skip in target.aliases]
+        if skipped:
+            exclude.append(skip)
+            display.warning('Excluding tests marked "%s" which are not yet supported on python 3: %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
     return exclude
 
 

test/runner/lib/thread.py

@@ -43,7 +43,7 @@ class WrappedThread(threading.Thread):
         result, exception = self._result.get()
         if exception:
             if sys.version_info[0] > 2:
-                raise exception[0](exception[1]).with_traceback(exception[2])
+                raise exception[1].with_traceback(exception[2])
             # noinspection PyRedundantParentheses
             exec('raise exception[0], exception[1], exception[2]')  # pylint: disable=locally-disabled, exec-used
         return result

test/runner/lib/util.py

@@ -208,7 +208,7 @@ def common_environment():
     return env
 
 
-def pass_vars(required=None, optional=None):
+def pass_vars(required, optional):
     """
     :type required: collections.Iterable[str]
     :type optional: collections.Iterable[str]

test/runner/tox.ini

@@ -5,5 +5,5 @@ minversion = 2.5.0
 [testenv]
 changedir = {toxinidir}/../../
 commands = {posargs}
-passenv = HOME
+passenv = HOME SHIPPABLE*
 args_are_paths = False

test/utils/shippable/network.sh

@@ -12,22 +12,38 @@ fi
 
 target="network/ci/"
 
+# python versions to test in order
+# all versions run full tests
+python_versions=(
+    2.7
+)
+
 if [ -s /tmp/network.txt ]; then
     echo "Detected changes requiring integration tests specific to networking:"
     cat /tmp/network.txt
 
     echo "Running network integration tests for multiple platforms concurrently."
 
-    # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-        --platform vyos/1.1.0 \
-        --platform ios/csr1000v \
-
+    platforms=(
+        --platform vyos/1.1.0
+        --platform ios/csr1000v
+    )
 else
     echo "No changes requiring integration tests specific to networking were detected."
     echo "Running network integration tests for a single platform only."
 
-    # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    platforms=(
         --platform vyos/1.1.0
+    )
 fi
+
+retry.py pip install tox --disable-pip-version-check
+
+for version in "${python_versions[@]}"; do
+    # clean up between test runs until we switch from --tox to --docker
+    rm -rf ~/.ansible/{cp,pc,tmp}/
+
+    # shellcheck disable=SC2086
+    ansible-test network-integration --color -v --retry-on-error "${target}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+        "${platforms[@]}"
+done

test/utils/shippable/windows.sh

@@ -7,6 +7,15 @@ IFS='/:' read -ra args <<< "${TEST}"
 
 job="${args[1]}"
 
+# python versions to test in order
+# python 2.7 runs full tests while other versions run minimal tests
+python_versions=(
+    2.6
+    3.5
+    3.6
+    2.7
+)
+
 # shellcheck disable=SC2086
 ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
 
@@ -23,20 +32,40 @@ if [ -s /tmp/windows.txt ]; then
 
     target="windows/ci/"
 
-    # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-        --windows 2008-SP2 \
-        --windows 2008-R2_SP1 \
-        --windows 2012-RTM \
-        --windows 2012-R2_RTM \
-
+    platforms=(
+        --windows 2008-SP2
+        --windows 2008-R2_SP1
+        --windows 2012-RTM
+        --windows 2012-R2_RTM
+    )
 else
     echo "No changes requiring integration tests specific to Windows were detected."
     echo "Running Windows integration tests for a single version only."
 
     target="windows/ci/group${job}/"
 
-    # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    platforms=(
         --windows 2012-R2_RTM
+    )
 fi
+
+retry.py pip install tox --disable-pip-version-check
+
+for version in "${python_versions[@]}"; do
+    # clean up between test runs until we switch from --tox to --docker
+    rm -rf ~/.ansible/{cp,pc,tmp}/
+
+    if [ "${job}" == "1" ] || [ "${version}" == "2.7" ]; then
+        if [ "${version}" == "2.7" ]; then
+            # full tests for python 2.7
+            ci="${target}"
+        else
+            # minimal tests for other python versions
+            ci="windows/ci/minimal/"
+        fi
+
+        # shellcheck disable=SC2086
+        ansible-test windows-integration --color -v --retry-on-error "${ci}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+            "${platforms[@]}"
+    fi
+done