From 0b784c65b1b52d2eef56d2dd5576be8702c14244 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 12 Jul 2017 20:04:22 -0700 Subject: [PATCH] Add sanity import test to ansible-test. (#26730) * Add sanity import test to ansible-test. * Run sanity import test on all python versions. --- test/runner/import/lib/ansible/__init__.py | 0 test/runner/import/lib/ansible/module_utils | 1 + test/runner/importer.py | 49 +++++++++ test/runner/injector/importer.py | 1 + test/runner/lib/executor.py | 5 +- test/runner/lib/sanity.py | 88 +++++++++++++++ test/runner/requirements/sanity.txt | 1 + test/sanity/import/skip.txt | 112 ++++++++++++++++++++ test/utils/shippable/other.sh | 4 +- 9 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 test/runner/import/lib/ansible/__init__.py create mode 120000 test/runner/import/lib/ansible/module_utils create mode 100755 test/runner/importer.py create mode 120000 test/runner/injector/importer.py create mode 100644 test/sanity/import/skip.txt diff --git a/test/runner/import/lib/ansible/__init__.py b/test/runner/import/lib/ansible/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/runner/import/lib/ansible/module_utils b/test/runner/import/lib/ansible/module_utils new file mode 120000 index 0000000000..fd236e25b5 --- /dev/null +++ b/test/runner/import/lib/ansible/module_utils @@ -0,0 +1 @@ +../../../../../lib/ansible/module_utils \ No newline at end of file diff --git a/test/runner/importer.py b/test/runner/importer.py new file mode 100755 index 0000000000..4a5861bd84 --- /dev/null +++ b/test/runner/importer.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +"""Import the given python module(s) and report error(s) encountered.""" + +from __future__ import absolute_import, print_function + +import imp +import os +import sys +import traceback + + +def main(): + """Main program function.""" + base_dir = os.getcwd() + messages = set() + + for path in sys.argv[1:]: + try: + with open(path, 'r') as module_fd: + imp.load_module('module_import_test', module_fd, os.path.abspath(path), ('.py', 'r', imp.PY_SOURCE)) + except Exception as ex: # pylint: disable=locally-disabled, broad-except + exc_type, _, exc_tb = sys.exc_info() + message = str(ex) + results = list(reversed(traceback.extract_tb(exc_tb))) + source = None + line = None + + for result in results: + if result[0].startswith(base_dir): + source = result[0][len(base_dir) + 1:].replace('test/runner/import/', '') + line = result[1] + break + + if not source: + source = path + line = 0 + message += ' (in %s:%d)' % (results[-1][0], results[-1][1]) + + error = '%s:%d:0: %s: %s' % (source, line, exc_type.__name__, message) + + if error not in messages: + messages.add(error) + print(error) + + if messages: + exit(10) + +if __name__ == '__main__': + main() diff --git a/test/runner/injector/importer.py b/test/runner/injector/importer.py new file mode 120000 index 0000000000..1f9d09cbf2 --- /dev/null +++ b/test/runner/injector/importer.py @@ -0,0 +1 @@ +injector.py \ No newline at end of file diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 286ad06169..a7b4bfcc06 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -864,7 +864,7 @@ def compile_version(args, python_version, include, exclude): return TestSuccess(command, test, python_version=python_version) -def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None): +def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None, path=None): """ :type args: TestConfig :type cmd: collections.Iterable[str] @@ -874,6 +874,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None :type data: str | None :type cwd: str | None :type python_version: str | None + :type path: str | None :rtype: str | None, str | None """ if not env: @@ -883,7 +884,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None inject_path = get_coverage_path(args) config_path = os.path.join(inject_path, 'injector.json') version = python_version or args.python_version - interpreter = find_executable('python%s' % version) + interpreter = find_executable('python%s' % version, path=path) coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % ( args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version))) diff --git a/test/runner/lib/sanity.py b/test/runner/lib/sanity.py index 473c9b0f85..04b40a38bb 100644 --- a/test/runner/lib/sanity.py +++ b/test/runner/lib/sanity.py @@ -19,6 +19,7 @@ from lib.util import ( run_command, deepest_path, parse_to_dict, + remove_tree, ) from lib.ansible_util import ( @@ -38,6 +39,7 @@ from lib.executor import ( install_command_requirements, SUPPORTED_PYTHON_VERSIONS, intercept_command, + generate_pip_install, ) from lib.config import ( @@ -668,6 +670,91 @@ def command_sanity_ansible_doc(args, targets, python_version): return SanitySuccess(test, python_version=python_version) +def command_sanity_import(args, targets, python_version): + """ + :type args: SanityConfig + :type targets: SanityTargets + :type python_version: str + :rtype: SanityResult + """ + test = 'import' + + with open('test/sanity/import/skip.txt', 'r') as skip_fd: + skip_paths = skip_fd.read().splitlines() + + skip_paths_set = set(skip_paths) + + paths = sorted( + i.path + for i in targets.include + if os.path.splitext(i.path)[1] == '.py' and + (i.path.startswith('lib/ansible/modules/') or i.path.startswith('lib/ansible/module_utils/')) and + i.path not in skip_paths_set + ) + + if not paths: + return SanitySkipped(test, python_version=python_version) + + env = ansible_environment(args, color=False) + + # create a clean virtual environment to minimize the available imports beyond the python standard library + virtual_environment_path = os.path.abspath('test/runner/.tox/minimal-py%s' % python_version.replace('.', '')) + virtual_environment_bin = os.path.join(virtual_environment_path, 'bin') + + remove_tree(virtual_environment_path) + + cmd = ['virtualenv', virtual_environment_path, '--python', 'python%s' % python_version, '--no-setuptools', '--no-wheel'] + + if not args.coverage: + cmd.append('--no-pip') + + run_command(args, cmd, capture=True) + + # add the importer to our virtual environment so it can be accessed through the coverage injector + importer_path = os.path.join(virtual_environment_bin, 'importer.py') + os.symlink(os.path.abspath('test/runner/importer.py'), importer_path) + + # activate the virtual environment + env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH']) + env['PYTHONPATH'] = os.path.abspath('test/runner/import/lib') + + # make sure coverage is available in the virtual environment if needed + if args.coverage: + run_command(args, generate_pip_install('sanity.import', packages=['coverage']), env=env) + run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env) + + cmd = ['importer.py'] + paths + + results = [] + + try: + stdout, stderr = intercept_command(args, cmd, target_name=test, env=env, capture=True, python_version=python_version, path=env['PATH']) + + if stdout or stderr: + raise SubprocessError(cmd, stdout=stdout, stderr=stderr) + except SubprocessError as ex: + if ex.status != 10 or ex.stderr or not ex.stdout: + raise + + pattern = r'^(?P[^:]*):(?P[0-9]+):(?P[0-9]+): (?P.*)$' + + results = [re.search(pattern, line).groupdict() for line in ex.stdout.splitlines()] + + results = [SanityMessage( + message=r['message'], + path=r['path'], + line=int(r['line']), + column=int(r['column']), + ) for r in results] + + results = [result for result in results if result.path not in skip_paths] + + if results: + return SanityFailure(test, messages=results, python_version=python_version) + + return SanitySuccess(test, python_version=python_version) + + def collect_code_smell_tests(): """ :rtype: tuple(SanityFunc) @@ -771,6 +858,7 @@ SANITY_TESTS = ( SanityFunc('rstcheck', command_sanity_rstcheck, intercept=False), SanityFunc('validate-modules', command_sanity_validate_modules, intercept=False), SanityFunc('ansible-doc', command_sanity_ansible_doc), + SanityFunc('import', command_sanity_import), ) diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt index adfeb47017..b87e34e189 100644 --- a/test/runner/requirements/sanity.txt +++ b/test/runner/requirements/sanity.txt @@ -7,5 +7,6 @@ pylint pytest rstcheck sphinx +virtualenv voluptuous yamllint diff --git a/test/sanity/import/skip.txt b/test/sanity/import/skip.txt new file mode 100644 index 0000000000..012d4054d4 --- /dev/null +++ b/test/sanity/import/skip.txt @@ -0,0 +1,112 @@ +lib/ansible/module_utils/ansible_tower.py +lib/ansible/module_utils/avi.py +lib/ansible/module_utils/azure_rm_common.py +lib/ansible/module_utils/ovirt.py +lib/ansible/module_utils/six/__init__.py +lib/ansible/modules/cloud/amazon/cloudformation.py +lib/ansible/modules/cloud/amazon/cloudtrail.py +lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_options.py +lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py +lib/ansible/modules/cloud/amazon/ec2_vpc_route_table.py +lib/ansible/modules/cloud/amazon/ec2_win_password.py +lib/ansible/modules/cloud/amazon/iam_cert_facts.py +lib/ansible/modules/cloud/amazon/s3_sync.py +lib/ansible/modules/cloud/azure/azure.py +lib/ansible/modules/cloud/centurylink/clc_firewall_policy.py +lib/ansible/modules/cloud/cloudstack/cs_account.py +lib/ansible/modules/cloud/cloudstack/cs_affinitygroup.py +lib/ansible/modules/cloud/cloudstack/cs_cluster.py +lib/ansible/modules/cloud/cloudstack/cs_domain.py +lib/ansible/modules/cloud/cloudstack/cs_facts.py +lib/ansible/modules/cloud/cloudstack/cs_firewall.py +lib/ansible/modules/cloud/cloudstack/cs_host.py +lib/ansible/modules/cloud/cloudstack/cs_instance_nic.py +lib/ansible/modules/cloud/cloudstack/cs_instancegroup.py +lib/ansible/modules/cloud/cloudstack/cs_ip_address.py +lib/ansible/modules/cloud/cloudstack/cs_iso.py +lib/ansible/modules/cloud/cloudstack/cs_loadbalancer_rule.py +lib/ansible/modules/cloud/cloudstack/cs_loadbalancer_rule_member.py +lib/ansible/modules/cloud/cloudstack/cs_network.py +lib/ansible/modules/cloud/cloudstack/cs_network_acl.py +lib/ansible/modules/cloud/cloudstack/cs_network_acl_rule.py +lib/ansible/modules/cloud/cloudstack/cs_pod.py +lib/ansible/modules/cloud/cloudstack/cs_project.py +lib/ansible/modules/cloud/cloudstack/cs_region.py +lib/ansible/modules/cloud/cloudstack/cs_resourcelimit.py +lib/ansible/modules/cloud/cloudstack/cs_role.py +lib/ansible/modules/cloud/cloudstack/cs_sshkeypair.py +lib/ansible/modules/cloud/cloudstack/cs_staticnat.py +lib/ansible/modules/cloud/cloudstack/cs_vpc.py +lib/ansible/modules/cloud/cloudstack/cs_vpn_gateway.py +lib/ansible/modules/cloud/cloudstack/cs_zone.py +lib/ansible/modules/cloud/cloudstack/cs_zone_facts.py +lib/ansible/modules/cloud/dimensiondata/dimensiondata_network.py +lib/ansible/modules/cloud/docker/docker_secret.py +lib/ansible/modules/cloud/google/gc_storage.py +lib/ansible/modules/cloud/google/gcdns_record.py +lib/ansible/modules/cloud/google/gcdns_zone.py +lib/ansible/modules/cloud/misc/serverless.py +lib/ansible/modules/cloud/openstack/os_client_config.py +lib/ansible/modules/cloud/openstack/os_ironic.py +lib/ansible/modules/cloud/ovirt/ovirt_disks.py +lib/ansible/modules/cloud/univention/udm_user.py +lib/ansible/modules/cloud/vmware/vca_nat.py +lib/ansible/modules/cloud/webfaction/webfaction_app.py +lib/ansible/modules/cloud/webfaction/webfaction_db.py +lib/ansible/modules/cloud/webfaction/webfaction_domain.py +lib/ansible/modules/cloud/webfaction/webfaction_mailbox.py +lib/ansible/modules/cloud/webfaction/webfaction_site.py +lib/ansible/modules/clustering/consul_acl.py +lib/ansible/modules/clustering/consul_kv.py +lib/ansible/modules/messaging/rabbitmq_binding.py +lib/ansible/modules/messaging/rabbitmq_exchange.py +lib/ansible/modules/messaging/rabbitmq_queue.py +lib/ansible/modules/monitoring/circonus_annotation.py +lib/ansible/modules/network/cloudengine/ce_file_copy.py +lib/ansible/modules/network/cumulus/_cl_img_install.py +lib/ansible/modules/network/f5/bigip_command.py +lib/ansible/modules/network/f5/bigip_config.py +lib/ansible/modules/network/f5/bigip_gtm_pool.py +lib/ansible/modules/network/f5/bigip_gtm_wide_ip.py +lib/ansible/modules/network/f5/bigip_hostname.py +lib/ansible/modules/network/f5/bigip_iapp_service.py +lib/ansible/modules/network/f5/bigip_iapp_template.py +lib/ansible/modules/network/f5/bigip_irule.py +lib/ansible/modules/network/f5/bigip_pool.py +lib/ansible/modules/network/f5/bigip_provision.py +lib/ansible/modules/network/f5/bigip_qkview.py +lib/ansible/modules/network/f5/bigip_snmp.py +lib/ansible/modules/network/f5/bigip_snmp_trap.py +lib/ansible/modules/network/f5/bigip_ssl_certificate.py +lib/ansible/modules/network/f5/bigip_user.py +lib/ansible/modules/network/f5/bigip_virtual_address.py +lib/ansible/modules/network/ios/ios_static_route.py +lib/ansible/modules/network/lenovo/cnos_backup.py +lib/ansible/modules/network/lenovo/cnos_bgp.py +lib/ansible/modules/network/lenovo/cnos_command.py +lib/ansible/modules/network/lenovo/cnos_conditional_command.py +lib/ansible/modules/network/lenovo/cnos_conditional_template.py +lib/ansible/modules/network/lenovo/cnos_factory.py +lib/ansible/modules/network/lenovo/cnos_facts.py +lib/ansible/modules/network/lenovo/cnos_image.py +lib/ansible/modules/network/lenovo/cnos_interface.py +lib/ansible/modules/network/lenovo/cnos_portchannel.py +lib/ansible/modules/network/lenovo/cnos_reload.py +lib/ansible/modules/network/lenovo/cnos_rollback.py +lib/ansible/modules/network/lenovo/cnos_save.py +lib/ansible/modules/network/lenovo/cnos_showrun.py +lib/ansible/modules/network/lenovo/cnos_template.py +lib/ansible/modules/network/lenovo/cnos_vlag.py +lib/ansible/modules/network/lenovo/cnos_vlan.py +lib/ansible/modules/network/nxos/nxos_file_copy.py +lib/ansible/modules/packaging/language/maven_artifact.py +lib/ansible/modules/packaging/os/rhn_channel.py +lib/ansible/modules/packaging/os/rhn_register.py +lib/ansible/modules/storage/infinidat/infini_export.py +lib/ansible/modules/storage/infinidat/infini_export_client.py +lib/ansible/modules/storage/infinidat/infini_fs.py +lib/ansible/modules/storage/infinidat/infini_host.py +lib/ansible/modules/storage/infinidat/infini_pool.py +lib/ansible/modules/storage/infinidat/infini_vol.py +lib/ansible/modules/utilities/helper/_accelerate.py +lib/ansible/modules/web_infrastructure/ansible_tower/tower_job_wait.py diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh index 5c436077ad..3f5b75316d 100755 --- a/test/utils/shippable/other.sh +++ b/test/utils/shippable/other.sh @@ -17,9 +17,9 @@ echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failur # shellcheck disable=SC2086 ansible-test compile --failure-ok --color -v --junit --requirements --coverage ${CHANGED:+"$CHANGED"} # shellcheck disable=SC2086 -ansible-test sanity --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage ${CHANGED:+"$CHANGED"} +ansible-test sanity --failure-ok --color -v --junit --tox --skip-test ansible-doc --skip-test import --python 3.5 --coverage ${CHANGED:+"$CHANGED"} # shellcheck disable=SC2086 -ansible-test sanity --failure-ok --color -v --junit --tox --test ansible-doc --coverage ${CHANGED:+"$CHANGED"} +ansible-test sanity --failure-ok --color -v --junit --tox --test ansible-doc --test import --coverage ${CHANGED:+"$CHANGED"} rm test/results/bot/ansible-test-failure.json