Add sanity import test to ansible-test. (#26730)

* Add sanity import test to ansible-test.
* Run sanity import test on all python versions.

test/runner/import/lib/ansible/module_utils

@@ -0,0 +1 @@
+../../../../../lib/ansible/module_utils

test/runner/importer.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""Import the given python module(s) and report error(s) encountered."""
+
+from __future__ import absolute_import, print_function
+
+import imp
+import os
+import sys
+import traceback
+
+
+def main():
+    """Main program function."""
+    base_dir = os.getcwd()
+    messages = set()
+
+    for path in sys.argv[1:]:
+        try:
+            with open(path, 'r') as module_fd:
+                imp.load_module('module_import_test', module_fd, os.path.abspath(path), ('.py', 'r', imp.PY_SOURCE))
+        except Exception as ex:  # pylint: disable=locally-disabled, broad-except
+            exc_type, _, exc_tb = sys.exc_info()
+            message = str(ex)
+            results = list(reversed(traceback.extract_tb(exc_tb)))
+            source = None
+            line = None
+
+            for result in results:
+                if result[0].startswith(base_dir):
+                    source = result[0][len(base_dir) + 1:].replace('test/runner/import/', '')
+                    line = result[1]
+                    break
+
+            if not source:
+                source = path
+                line = 0
+                message += ' (in %s:%d)' % (results[-1][0], results[-1][1])
+
+            error = '%s:%d:0: %s: %s' % (source, line, exc_type.__name__, message)
+
+            if error not in messages:
+                messages.add(error)
+                print(error)
+
+    if messages:
+        exit(10)
+
+if __name__ == '__main__':
+    main()

test/runner/injector/importer.py

@@ -0,0 +1 @@
+injector.py

test/runner/lib/executor.py

@@ -864,7 +864,7 @@ def compile_version(args, python_version, include, exclude):
     return TestSuccess(command, test, python_version=python_version)
 
 
-def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None):
+def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None, path=None):
     """
     :type args: TestConfig
     :type cmd: collections.Iterable[str]
@@ -874,6 +874,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None
     :type data: str | None
     :type cwd: str | None
     :type python_version: str | None
+    :type path: str | None
     :rtype: str | None, str | None
     """
     if not env:
@@ -883,7 +884,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None
     inject_path = get_coverage_path(args)
     config_path = os.path.join(inject_path, 'injector.json')
     version = python_version or args.python_version
-    interpreter = find_executable('python%s' % version)
+    interpreter = find_executable('python%s' % version, path=path)
     coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % (
         args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)))
 

test/runner/lib/sanity.py

@@ -19,6 +19,7 @@ from lib.util import (
     run_command,
     deepest_path,
     parse_to_dict,
+    remove_tree,
 )
 
 from lib.ansible_util import (
@@ -38,6 +39,7 @@ from lib.executor import (
     install_command_requirements,
     SUPPORTED_PYTHON_VERSIONS,
     intercept_command,
+    generate_pip_install,
 )
 
 from lib.config import (
@@ -668,6 +670,91 @@ def command_sanity_ansible_doc(args, targets, python_version):
     return SanitySuccess(test, python_version=python_version)
 
 
+def command_sanity_import(args, targets, python_version):
+    """
+    :type args: SanityConfig
+    :type targets: SanityTargets
+    :type python_version: str
+    :rtype: SanityResult
+    """
+    test = 'import'
+
+    with open('test/sanity/import/skip.txt', 'r') as skip_fd:
+        skip_paths = skip_fd.read().splitlines()
+
+    skip_paths_set = set(skip_paths)
+
+    paths = sorted(
+        i.path
+        for i in targets.include
+        if os.path.splitext(i.path)[1] == '.py' and
+        (i.path.startswith('lib/ansible/modules/') or i.path.startswith('lib/ansible/module_utils/')) and
+        i.path not in skip_paths_set
+    )
+
+    if not paths:
+        return SanitySkipped(test, python_version=python_version)
+
+    env = ansible_environment(args, color=False)
+
+    # create a clean virtual environment to minimize the available imports beyond the python standard library
+    virtual_environment_path = os.path.abspath('test/runner/.tox/minimal-py%s' % python_version.replace('.', ''))
+    virtual_environment_bin = os.path.join(virtual_environment_path, 'bin')
+
+    remove_tree(virtual_environment_path)
+
+    cmd = ['virtualenv', virtual_environment_path, '--python', 'python%s' % python_version, '--no-setuptools', '--no-wheel']
+
+    if not args.coverage:
+        cmd.append('--no-pip')
+
+    run_command(args, cmd, capture=True)
+
+    # add the importer to our virtual environment so it can be accessed through the coverage injector
+    importer_path = os.path.join(virtual_environment_bin, 'importer.py')
+    os.symlink(os.path.abspath('test/runner/importer.py'), importer_path)
+
+    # activate the virtual environment
+    env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH'])
+    env['PYTHONPATH'] = os.path.abspath('test/runner/import/lib')
+
+    # make sure coverage is available in the virtual environment if needed
+    if args.coverage:
+        run_command(args, generate_pip_install('sanity.import', packages=['coverage']), env=env)
+        run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env)
+
+    cmd = ['importer.py'] + paths
+
+    results = []
+
+    try:
+        stdout, stderr = intercept_command(args, cmd, target_name=test, env=env, capture=True, python_version=python_version, path=env['PATH'])
+
+        if stdout or stderr:
+            raise SubprocessError(cmd, stdout=stdout, stderr=stderr)
+    except SubprocessError as ex:
+        if ex.status != 10 or ex.stderr or not ex.stdout:
+            raise
+
+        pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$'
+
+        results = [re.search(pattern, line).groupdict() for line in ex.stdout.splitlines()]
+
+        results = [SanityMessage(
+            message=r['message'],
+            path=r['path'],
+            line=int(r['line']),
+            column=int(r['column']),
+        ) for r in results]
+
+        results = [result for result in results if result.path not in skip_paths]
+
+    if results:
+        return SanityFailure(test, messages=results, python_version=python_version)
+
+    return SanitySuccess(test, python_version=python_version)
+
+
 def collect_code_smell_tests():
     """
     :rtype: tuple(SanityFunc)
@@ -771,6 +858,7 @@ SANITY_TESTS = (
     SanityFunc('rstcheck', command_sanity_rstcheck, intercept=False),
     SanityFunc('validate-modules', command_sanity_validate_modules, intercept=False),
     SanityFunc('ansible-doc', command_sanity_ansible_doc),
+    SanityFunc('import', command_sanity_import),
 )
 
 

test/runner/requirements/sanity.txt

@@ -7,5 +7,6 @@ pylint
 pytest
 rstcheck
 sphinx
+virtualenv
 voluptuous
 yamllint

test/sanity/import/skip.txt

@@ -0,0 +1,112 @@
+lib/ansible/module_utils/ansible_tower.py
+lib/ansible/module_utils/avi.py
+lib/ansible/module_utils/azure_rm_common.py
+lib/ansible/module_utils/ovirt.py
+lib/ansible/module_utils/six/__init__.py
+lib/ansible/modules/cloud/amazon/cloudformation.py
+lib/ansible/modules/cloud/amazon/cloudtrail.py
+lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_options.py
+lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py
+lib/ansible/modules/cloud/amazon/ec2_vpc_route_table.py
+lib/ansible/modules/cloud/amazon/ec2_win_password.py
+lib/ansible/modules/cloud/amazon/iam_cert_facts.py
+lib/ansible/modules/cloud/amazon/s3_sync.py
+lib/ansible/modules/cloud/azure/azure.py
+lib/ansible/modules/cloud/centurylink/clc_firewall_policy.py
+lib/ansible/modules/cloud/cloudstack/cs_account.py
+lib/ansible/modules/cloud/cloudstack/cs_affinitygroup.py
+lib/ansible/modules/cloud/cloudstack/cs_cluster.py
+lib/ansible/modules/cloud/cloudstack/cs_domain.py
+lib/ansible/modules/cloud/cloudstack/cs_facts.py
+lib/ansible/modules/cloud/cloudstack/cs_firewall.py
+lib/ansible/modules/cloud/cloudstack/cs_host.py
+lib/ansible/modules/cloud/cloudstack/cs_instance_nic.py
+lib/ansible/modules/cloud/cloudstack/cs_instancegroup.py
+lib/ansible/modules/cloud/cloudstack/cs_ip_address.py
+lib/ansible/modules/cloud/cloudstack/cs_iso.py
+lib/ansible/modules/cloud/cloudstack/cs_loadbalancer_rule.py
+lib/ansible/modules/cloud/cloudstack/cs_loadbalancer_rule_member.py
+lib/ansible/modules/cloud/cloudstack/cs_network.py
+lib/ansible/modules/cloud/cloudstack/cs_network_acl.py
+lib/ansible/modules/cloud/cloudstack/cs_network_acl_rule.py
+lib/ansible/modules/cloud/cloudstack/cs_pod.py
+lib/ansible/modules/cloud/cloudstack/cs_project.py
+lib/ansible/modules/cloud/cloudstack/cs_region.py
+lib/ansible/modules/cloud/cloudstack/cs_resourcelimit.py
+lib/ansible/modules/cloud/cloudstack/cs_role.py
+lib/ansible/modules/cloud/cloudstack/cs_sshkeypair.py
+lib/ansible/modules/cloud/cloudstack/cs_staticnat.py
+lib/ansible/modules/cloud/cloudstack/cs_vpc.py
+lib/ansible/modules/cloud/cloudstack/cs_vpn_gateway.py
+lib/ansible/modules/cloud/cloudstack/cs_zone.py
+lib/ansible/modules/cloud/cloudstack/cs_zone_facts.py
+lib/ansible/modules/cloud/dimensiondata/dimensiondata_network.py
+lib/ansible/modules/cloud/docker/docker_secret.py
+lib/ansible/modules/cloud/google/gc_storage.py
+lib/ansible/modules/cloud/google/gcdns_record.py
+lib/ansible/modules/cloud/google/gcdns_zone.py
+lib/ansible/modules/cloud/misc/serverless.py
+lib/ansible/modules/cloud/openstack/os_client_config.py
+lib/ansible/modules/cloud/openstack/os_ironic.py
+lib/ansible/modules/cloud/ovirt/ovirt_disks.py
+lib/ansible/modules/cloud/univention/udm_user.py
+lib/ansible/modules/cloud/vmware/vca_nat.py
+lib/ansible/modules/cloud/webfaction/webfaction_app.py
+lib/ansible/modules/cloud/webfaction/webfaction_db.py
+lib/ansible/modules/cloud/webfaction/webfaction_domain.py
+lib/ansible/modules/cloud/webfaction/webfaction_mailbox.py
+lib/ansible/modules/cloud/webfaction/webfaction_site.py
+lib/ansible/modules/clustering/consul_acl.py
+lib/ansible/modules/clustering/consul_kv.py
+lib/ansible/modules/messaging/rabbitmq_binding.py
+lib/ansible/modules/messaging/rabbitmq_exchange.py
+lib/ansible/modules/messaging/rabbitmq_queue.py
+lib/ansible/modules/monitoring/circonus_annotation.py
+lib/ansible/modules/network/cloudengine/ce_file_copy.py
+lib/ansible/modules/network/cumulus/_cl_img_install.py
+lib/ansible/modules/network/f5/bigip_command.py
+lib/ansible/modules/network/f5/bigip_config.py
+lib/ansible/modules/network/f5/bigip_gtm_pool.py
+lib/ansible/modules/network/f5/bigip_gtm_wide_ip.py
+lib/ansible/modules/network/f5/bigip_hostname.py
+lib/ansible/modules/network/f5/bigip_iapp_service.py
+lib/ansible/modules/network/f5/bigip_iapp_template.py
+lib/ansible/modules/network/f5/bigip_irule.py
+lib/ansible/modules/network/f5/bigip_pool.py
+lib/ansible/modules/network/f5/bigip_provision.py
+lib/ansible/modules/network/f5/bigip_qkview.py
+lib/ansible/modules/network/f5/bigip_snmp.py
+lib/ansible/modules/network/f5/bigip_snmp_trap.py
+lib/ansible/modules/network/f5/bigip_ssl_certificate.py
+lib/ansible/modules/network/f5/bigip_user.py
+lib/ansible/modules/network/f5/bigip_virtual_address.py
+lib/ansible/modules/network/ios/ios_static_route.py
+lib/ansible/modules/network/lenovo/cnos_backup.py
+lib/ansible/modules/network/lenovo/cnos_bgp.py
+lib/ansible/modules/network/lenovo/cnos_command.py
+lib/ansible/modules/network/lenovo/cnos_conditional_command.py
+lib/ansible/modules/network/lenovo/cnos_conditional_template.py
+lib/ansible/modules/network/lenovo/cnos_factory.py
+lib/ansible/modules/network/lenovo/cnos_facts.py
+lib/ansible/modules/network/lenovo/cnos_image.py
+lib/ansible/modules/network/lenovo/cnos_interface.py
+lib/ansible/modules/network/lenovo/cnos_portchannel.py
+lib/ansible/modules/network/lenovo/cnos_reload.py
+lib/ansible/modules/network/lenovo/cnos_rollback.py
+lib/ansible/modules/network/lenovo/cnos_save.py
+lib/ansible/modules/network/lenovo/cnos_showrun.py
+lib/ansible/modules/network/lenovo/cnos_template.py
+lib/ansible/modules/network/lenovo/cnos_vlag.py
+lib/ansible/modules/network/lenovo/cnos_vlan.py
+lib/ansible/modules/network/nxos/nxos_file_copy.py
+lib/ansible/modules/packaging/language/maven_artifact.py
+lib/ansible/modules/packaging/os/rhn_channel.py
+lib/ansible/modules/packaging/os/rhn_register.py
+lib/ansible/modules/storage/infinidat/infini_export.py
+lib/ansible/modules/storage/infinidat/infini_export_client.py
+lib/ansible/modules/storage/infinidat/infini_fs.py
+lib/ansible/modules/storage/infinidat/infini_host.py
+lib/ansible/modules/storage/infinidat/infini_pool.py
+lib/ansible/modules/storage/infinidat/infini_vol.py
+lib/ansible/modules/utilities/helper/_accelerate.py
+lib/ansible/modules/web_infrastructure/ansible_tower/tower_job_wait.py

test/utils/shippable/other.sh

@@ -17,9 +17,9 @@ echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failur
 # shellcheck disable=SC2086
 ansible-test compile --failure-ok --color -v --junit --requirements --coverage ${CHANGED:+"$CHANGED"}
 # shellcheck disable=SC2086
-ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage ${CHANGED:+"$CHANGED"}
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --skip-test import --python 3.5 --coverage ${CHANGED:+"$CHANGED"}
 # shellcheck disable=SC2086
-ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage ${CHANGED:+"$CHANGED"}
+ansible-test sanity  --failure-ok --color -v --junit --tox --test      ansible-doc --test      import              --coverage ${CHANGED:+"$CHANGED"}
 
 rm test/results/bot/ansible-test-failure.json