diff --git a/test/runner/lib/cloud/aws.py b/test/runner/lib/cloud/aws.py index 73cec7ed28..77937ccb6f 100644 --- a/test/runner/lib/cloud/aws.py +++ b/test/runner/lib/cloud/aws.py @@ -79,7 +79,7 @@ class AwsCloudProvider(CloudProvider): """ :rtype: AnsibleCoreCI """ - return AnsibleCoreCI(self.args, 'aws', 'sts', persist=False, stage=self.args.remote_stage) + return AnsibleCoreCI(self.args, 'aws', 'sts', persist=False, stage=self.args.remote_stage, provider=self.args.remote_provider) class AwsCloudEnvironment(CloudEnvironment): diff --git a/test/runner/lib/cloud/azure.py b/test/runner/lib/cloud/azure.py index 97e34e2969..3597b21a9a 100644 --- a/test/runner/lib/cloud/azure.py +++ b/test/runner/lib/cloud/azure.py @@ -114,7 +114,7 @@ class AzureCloudProvider(CloudProvider): """ :rtype: AnsibleCoreCI """ - return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage) + return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage, provider=self.args.remote_provider) class AzureCloudEnvironment(CloudEnvironment): diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index 0bf8b9581d..bdec3f3976 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -48,9 +48,13 @@ class EnvironmentConfig(CommonConfig): self.tox_sitepackages = args.tox_sitepackages # type: bool self.remote_stage = args.remote_stage # type: str + self.remote_provider = args.remote_provider # type: str self.remote_aws_region = args.remote_aws_region # type: str self.remote_terminate = args.remote_terminate # type: str + if self.remote_provider == 'default': + self.remote_provider = None + self.requirements = args.requirements # type: bool if self.python == 'default': diff --git a/test/runner/lib/core_ci.py b/test/runner/lib/core_ci.py index 4746fa4e86..4867956862 100644 --- a/test/runner/lib/core_ci.py +++ b/test/runner/lib/core_ci.py @@ -36,7 +36,7 @@ AWS_ENDPOINTS = { class AnsibleCoreCI(object): """Client for Ansible Core CI services.""" - def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None): + def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None, provider=None): """ :type args: EnvironmentConfig :type platform: str @@ -57,23 +57,42 @@ class AnsibleCoreCI(object): self.max_threshold = 1 self.name = name if name else '%s-%s' % (self.platform, self.version) self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') + self.resource = 'jobs' - aws_platforms = ( - 'aws', - 'azure', - 'windows', - 'freebsd', - 'rhel', - 'vyos', - 'junos', - 'ios', + # Assign each supported platform to one provider. + # This is used to determine the provider from the platform when no provider is specified. + providers = dict( + aws=( + 'aws', + 'azure', + 'windows', + 'freebsd', + 'rhel', + 'vyos', + 'junos', + 'ios', + ), + azure=( + ), + parallels=( + 'osx', + ), ) - osx_platforms = ( - 'osx', - ) + if provider: + # override default provider selection (not all combinations are valid) + self.provider = provider + else: + for candidate in providers: + if platform in providers[candidate]: + # assign default provider based on platform + self.provider = candidate + break + + if self.provider in ('aws', 'azure'): + if self.provider != 'aws': + self.resource = self.provider - if self.platform in aws_platforms: if args.remote_aws_region: # permit command-line override of region selection region = args.remote_aws_region @@ -97,7 +116,7 @@ class AnsibleCoreCI(object): else: self.ssh_key = SshKey(args) self.port = 22 - elif self.platform in osx_platforms: + elif self.provider == 'parallels': self.endpoints = self._get_parallels_endpoints() self.max_threshold = 6 @@ -106,7 +125,7 @@ class AnsibleCoreCI(object): else: raise ApplicationError('Unsupported platform: %s' % platform) - self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage)) + self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage)) if persist and load and self._load(): try: @@ -292,7 +311,7 @@ class AnsibleCoreCI(object): @property def _uri(self): - return '%s/%s/jobs/%s' % (self.endpoint, self.stage, self.instance_id) + return '%s/%s/%s/%s' % (self.endpoint, self.stage, self.resource, self.instance_id) def _start(self, auth): """Start instance.""" diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index 55ac881eba..2ba4b24f08 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -280,7 +280,7 @@ def delegate_remote(args, exclude, require): platform = parts[0] version = parts[1] - core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage) + core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) success = False try: diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index f563730637..2d0049d8a3 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -406,7 +406,7 @@ def network_start(args, platform, version): :type version: str :rtype: AnsibleCoreCI """ - core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage) + core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) core_ci.start() return core_ci.save() @@ -420,7 +420,7 @@ def network_run(args, platform, version, config): :type config: dict[str, str] :rtype: AnsibleCoreCI """ - core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, load=False) + core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider, load=False) core_ci.load(config) core_ci.wait() @@ -548,7 +548,7 @@ def windows_start(args, version): :type version: str :rtype: AnsibleCoreCI """ - core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage) + core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage, provider=args.remote_provider) core_ci.start() return core_ci.save() @@ -561,7 +561,7 @@ def windows_run(args, version, config): :type config: dict[str, str] :rtype: AnsibleCoreCI """ - core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage, load=False) + core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage, provider=args.remote_provider, load=False) core_ci.load(config) core_ci.wait() diff --git a/test/runner/lib/manage_ci.py b/test/runner/lib/manage_ci.py index 0138f5654c..14d4625b94 100644 --- a/test/runner/lib/manage_ci.py +++ b/test/runner/lib/manage_ci.py @@ -131,7 +131,12 @@ class ManagePosixCI(object): self.ssh_args += ['-o', '%s=%s' % (ssh_option, ssh_options[ssh_option])] if self.core_ci.platform == 'freebsd': - self.become = ['su', '-l', 'root', '-c'] + if self.core_ci.provider == 'aws': + self.become = ['su', '-l', 'root', '-c'] + elif self.core_ci.provider == 'azure': + self.become = ['sudo', '-in', 'sh', '-c'] + else: + raise NotImplementedError('provider %s has not been implemented' % self.core_ci.provider) elif self.core_ci.platform == 'osx': self.become = ['sudo', '-in', 'PATH=/usr/local/bin:$PATH'] elif self.core_ci.platform == 'rhel': diff --git a/test/runner/setup/remote.sh b/test/runner/setup/remote.sh index f48fdbd7a1..2b83056c81 100644 --- a/test/runner/setup/remote.sh +++ b/test/runner/setup/remote.sh @@ -10,6 +10,7 @@ cd ~/ if [ "${platform}" = "freebsd" ]; then while true; do + env ASSUME_ALWAYS_YES=YES pkg bootstrap && \ pkg install -y \ bash \ curl \ diff --git a/test/runner/test.py b/test/runner/test.py index 5337f4c82d..8f0f282b65 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -514,6 +514,7 @@ def add_environments(parser, tox_version=False, tox_only=False): docker=None, remote=None, remote_stage=None, + remote_provider=None, remote_aws_region=None, remote_terminate=None, ) @@ -540,6 +541,12 @@ def add_environments(parser, tox_version=False, tox_only=False): choices=['prod', 'dev'], default='prod') + remote.add_argument('--remote-provider', + metavar='PROVIDER', + help='remote provider to use: %(choices)s', + choices=['default', 'aws', 'azure', 'parallels'], + default='default') + remote.add_argument('--remote-aws-region', metavar='REGION', help='remote aws region to use: %(choices)s (default: auto)', diff --git a/test/utils/shippable/freebsd.sh b/test/utils/shippable/freebsd.sh index 603b1a08c4..23ea98d5ca 100755 --- a/test/utils/shippable/freebsd.sh +++ b/test/utils/shippable/freebsd.sh @@ -14,7 +14,10 @@ else target="posix/ci/" fi +stage="${S:-prod}" +provider="${P:-default}" + # shellcheck disable=SC2086 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --exclude "posix/ci/cloud/" \ - --remote "${platform}/${version}" --remote-terminate always + --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh index 540396e45d..cea47078f4 100755 --- a/test/utils/shippable/network.sh +++ b/test/utils/shippable/network.sh @@ -12,6 +12,9 @@ fi target="network/ci/" +stage="${S:-prod}" +provider="${P:-default}" + # python versions to test in order # all versions run full tests python_versions=( @@ -50,5 +53,6 @@ for version in "${python_versions[@]}"; do # shellcheck disable=SC2086 ansible-test network-integration --color -v --retry-on-error "${target}" --docker default --python "${version}" \ - ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} "${platforms[@]}" --remote-terminate "${terminate}" + ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} "${platforms[@]}" \ + --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}" done diff --git a/test/utils/shippable/osx.sh b/test/utils/shippable/osx.sh index 603b1a08c4..23ea98d5ca 100755 --- a/test/utils/shippable/osx.sh +++ b/test/utils/shippable/osx.sh @@ -14,7 +14,10 @@ else target="posix/ci/" fi +stage="${S:-prod}" +provider="${P:-default}" + # shellcheck disable=SC2086 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --exclude "posix/ci/cloud/" \ - --remote "${platform}/${version}" --remote-terminate always + --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" diff --git a/test/utils/shippable/rhel.sh b/test/utils/shippable/rhel.sh index 603b1a08c4..23ea98d5ca 100755 --- a/test/utils/shippable/rhel.sh +++ b/test/utils/shippable/rhel.sh @@ -14,7 +14,10 @@ else target="posix/ci/" fi +stage="${S:-prod}" +provider="${P:-default}" + # shellcheck disable=SC2086 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --exclude "posix/ci/cloud/" \ - --remote "${platform}/${version}" --remote-terminate always + --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh index 9d4a82c096..b3f9f5d1b4 100755 --- a/test/utils/shippable/windows.sh +++ b/test/utils/shippable/windows.sh @@ -7,6 +7,9 @@ IFS='/:' read -ra args <<< "$1" target="windows/ci/group${args[1]}/" +stage="${S:-prod}" +provider="${P:-default}" + # python versions to test in order # python 2.7 runs full tests while other versions run minimal tests python_versions=( @@ -76,5 +79,6 @@ for version in "${python_versions[@]}"; do # shellcheck disable=SC2086 ansible-test windows-integration --color -v --retry-on-error "${ci}" --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ - "${platforms[@]}" --changed-all-target "${changed_all_target}" --remote-terminate "${terminate}" + "${platforms[@]}" --changed-all-target "${changed_all_target}" \ + --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}" done