From e53390b3b1afedc475294f25b9c2b847fa63a806 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 2 Oct 2018 12:26:14 -0700 Subject: [PATCH] Fix ansible-test multi-group smoke test handling. (#46363) * Fix ansible-test smoke tests across groups. * Fix ansible-test list arg defaults. * Fix ansible-test require and exclude delegation. * Fix detection of Windows specific changes. * Add minimal Windows testing for Python 3.7. --- test/runner/lib/cli.py | 10 ++++++++++ test/runner/lib/config.py | 8 +++++--- test/runner/lib/delegation.py | 5 +++++ test/runner/lib/executor.py | 19 +++++++++++++++---- test/runner/lib/sanity/__init__.py | 2 +- test/utils/shippable/cloud.sh | 8 +++++--- test/utils/shippable/windows.sh | 11 +++++++---- 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py index b32887807d..a5ebf1cd40 100644 --- a/test/runner/lib/cli.py +++ b/test/runner/lib/cli.py @@ -169,6 +169,11 @@ def parse_args(): nargs='*', help='test the specified target').completer = complete_target + test.add_argument('--include', + metavar='TARGET', + action='append', + help='include the specified target').completer = complete_target + test.add_argument('--exclude', metavar='TARGET', action='append', @@ -261,6 +266,11 @@ def parse_args(): default='all', help='target to run when all tests are needed') + integration.add_argument('--changed-all-mode', + metavar='MODE', + choices=('default', 'include', 'exclude'), + help='include/exclude behavior with --changed-all-target: %(choices)s') + integration.add_argument('--list-targets', action='store_true', help='list matching targets instead of running tests') diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index eeb179cc39..fce8326a33 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -71,6 +71,7 @@ class EnvironmentConfig(CommonConfig): self.python_version = self.python or '.'.join(str(i) for i in sys.version_info[:2]) self.delegate = self.tox or self.docker or self.remote + self.delegate_args = [] # type: list[str] if self.delegate: self.requirements = True @@ -104,9 +105,9 @@ class TestConfig(EnvironmentConfig): self.coverage = args.coverage # type: bool self.coverage_label = args.coverage_label # type: str - self.include = args.include # type: list [str] - self.exclude = args.exclude # type: list [str] - self.require = args.require # type: list [str] + self.include = args.include or [] # type: list [str] + self.exclude = args.exclude or [] # type: list [str] + self.require = args.require or [] # type: list [str] self.changed = args.changed # type: bool self.tracked = args.tracked # type: bool @@ -179,6 +180,7 @@ class IntegrationConfig(TestConfig): self.continue_on_error = args.continue_on_error # type: bool self.debug_strategy = args.debug_strategy # type: bool self.changed_all_target = args.changed_all_target # type: str + self.changed_all_mode = args.changed_all_mode # type: str self.list_targets = args.list_targets # type: bool self.tags = args.tags self.skip_tags = args.skip_tags diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index aea242a640..0932cfcac6 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -458,6 +458,8 @@ def filter_options(args, argv, options, exclude, require): '--changed-from': 1, '--changed-path': 1, '--metadata': 1, + '--exclude': 1, + '--require': 1, }) elif isinstance(args, SanityConfig): options.update({ @@ -482,6 +484,9 @@ def filter_options(args, argv, options, exclude, require): yield arg + for arg in args.delegate_args: + yield arg + for target in exclude: yield '--exclude' yield target diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index ac8bcd4bdb..8fce375285 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -651,8 +651,19 @@ def command_integration_filter(args, targets, init_callback=None): """ targets = tuple(target for target in targets if 'hidden/' not in target.aliases) changes = get_changes_filter(args) - require = (args.require or []) + changes - exclude = (args.exclude or []) + + # special behavior when the --changed-all-target target is selected based on changes + if args.changed_all_target in changes: + # act as though the --changed-all-target target was in the include list + if args.changed_all_mode == 'include' and args.changed_all_target not in args.include: + args.include.append(args.changed_all_target) + args.delegate_args += ['--include', args.changed_all_target] + # act as though the --changed-all-target target was in the exclude list + elif args.changed_all_mode == 'exclude' and args.changed_all_target not in args.exclude: + args.exclude.append(args.changed_all_target) + + require = args.require + changes + exclude = args.exclude internal_targets = walk_internal_targets(targets, args.include, exclude, require) environment_exclude = get_integration_filter(args, internal_targets) @@ -675,7 +686,7 @@ def command_integration_filter(args, targets, init_callback=None): cloud_init(args, internal_targets) if args.delegate: - raise Delegate(require=changes, exclude=exclude, integration_targets=internal_targets) + raise Delegate(require=require, exclude=exclude, integration_targets=internal_targets) install_command_requirements(args) @@ -1133,7 +1144,7 @@ def command_units(args): :type args: UnitsConfig """ changes = get_changes_filter(args) - require = (args.require or []) + changes + require = args.require + changes include, exclude = walk_external_targets(walk_units_targets(), args.include, args.exclude, require) if not include: diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py index beda485a5f..ed4b0a5fec 100644 --- a/test/runner/lib/sanity/__init__.py +++ b/test/runner/lib/sanity/__init__.py @@ -58,7 +58,7 @@ def command_sanity(args): :type args: SanityConfig """ changes = get_changes_filter(args) - require = (args.require or []) + changes + require = args.require + changes targets = SanityTargets(args.include, args.exclude, require) if not targets.include: diff --git a/test/utils/shippable/cloud.sh b/test/utils/shippable/cloud.sh index 46ed859057..06a554b8f9 100755 --- a/test/utils/shippable/cloud.sh +++ b/test/utils/shippable/cloud.sh @@ -13,9 +13,11 @@ target="shippable/${cloud}/group${group}/" stage="${S:-prod}" +changed_all_target="shippable/${cloud}/smoketest/" + if [ "${group}" == "1" ]; then # only run smoketest tests for group1 - changed_all_target="shippable/${cloud}/smoketest/" + changed_all_mode="include" if ! ansible-test integration "${changed_all_target}" --list-targets > /dev/null 2>&1; then # no smoketest tests are available for this cloud @@ -23,10 +25,10 @@ if [ "${group}" == "1" ]; then fi else # smoketest tests already covered by group1 - changed_all_target="none" + changed_all_mode="exclude" fi # shellcheck disable=SC2086 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \ --remote-terminate always --remote-stage "${stage}" \ - --docker --python "${python}" --changed-all-target "${changed_all_target}" + --docker --python "${python}" --changed-all-target "${changed_all_target}" --changed-all-mode "${changed_all_mode}" diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh index 8b19388dfc..45653cb505 100755 --- a/test/utils/shippable/windows.sh +++ b/test/utils/shippable/windows.sh @@ -19,6 +19,7 @@ python_versions=( 2.6 3.5 3.6 + 3.7 2.7 ) @@ -26,7 +27,7 @@ python_versions=( single_version=2012-R2 # shellcheck disable=SC2086 -ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \ +ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \ | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt if [ -s /tmp/windows.txt ] || [ "${CHANGED:+$CHANGED}" == "" ]; then @@ -54,6 +55,7 @@ fi for version in "${python_versions[@]}"; do changed_all_target="all" + changed_all_mode="default" if [ "${version}" == "2.7" ]; then # smoketest tests for python 2.7 @@ -61,12 +63,13 @@ for version in "${python_versions[@]}"; do # with change detection enabled run tests for anything changed # use the smoketest tests for any change that triggers all tests ci="${target}" + changed_all_target="shippable/windows/smoketest/" if [ "${target}" == "shippable/windows/group1/" ]; then # only run smoketest tests for group1 - changed_all_target="shippable/windows/smoketest/" + changed_all_mode="include" else # smoketest tests already covered by group1 - changed_all_target="none" + changed_all_mode="exclude" fi else # without change detection enabled run entire test group @@ -88,7 +91,7 @@ for version in "${python_versions[@]}"; do # shellcheck disable=SC2086 ansible-test windows-integration --color -v --retry-on-error "${ci}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \ - "${platforms[@]}" --changed-all-target "${changed_all_target}" \ + "${platforms[@]}" --changed-all-target "${changed_all_target}" --changed-all-mode "${changed_all_mode}" \ --docker default --python "${version}" \ --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}" done