"""Delegate test execution to another environment."""
from __future__ import absolute_import, print_function
import os
import sys
import lib.pytar
import lib.thread
from lib.executor import (
SUPPORTED_PYTHON_VERSIONS,
EnvironmentConfig,
IntegrationConfig,
ShellConfig,
TestConfig,
create_shell_command,
)
from lib.core_ci import (
AnsibleCoreCI,
)
from lib.manage_ci import (
ManagePosixCI,
)
from lib.util import (
ApplicationError,
run_command,
)
BUFFER_SIZE = 256 * 256
def delegate(args, exclude, require):
"""
:type args: EnvironmentConfig
:type exclude: list[str]
:type require: list[str]
"""
if args.tox:
delegate_tox(args, exclude, require)
return True
if args.docker:
delegate_docker(args, exclude, require)
return True
if args.remote:
delegate_remote(args, exclude, require)
return True
return False
def delegate_tox(args, exclude, require):
"""
:type args: EnvironmentConfig
:type exclude: list[str]
:type require: list[str]
"""
if args.python:
versions = args.python,
if args.python not in SUPPORTED_PYTHON_VERSIONS:
raise ApplicationError('tox does not support Python version %s' % args.python)
else:
versions = SUPPORTED_PYTHON_VERSIONS
options = {
'--tox': args.tox_args,
}
for version in versions:
tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', ''), '--']
cmd = generate_command(args, os.path.abspath('test/runner/test.py'), options, exclude, require)
if not args.python:
cmd += ['--python', version]
run_command(args, tox + cmd)
def delegate_docker(args, exclude, require):
"""
:type args: EnvironmentConfig
:type exclude: list[str]
:type require: list[str]
"""
util_image = args.docker_util
test_image = args.docker
privileged = args.docker_privileged
util_id = None
test_id = None
options = {
'--docker': 1,
'--docker-privileged': 0,
'--docker-util': 1,
}
cmd = generate_command(args, '/root/ansible/test/runner/test.py', options, exclude, require)
if isinstance(args, IntegrationConfig):
if not args.allow_destructive:
cmd.append('--allow-destructive')
if not args.explain:
lib.pytar.create_tarfile('/tmp/ansible.tgz', '.', lib.pytar.ignore)
try:
if util_image:
util_id, _ = run_command(args, [
'docker', 'run', '--detach',
util_image,
], capture=True)
if args.explain:
util_id = 'util_id'
else:
util_id = util_id.strip()
else:
util_id = None
test_cmd = [
'docker', 'run', '--detach',
'--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro',
'--privileged=%s' % str(privileged).lower(),
]
if util_id:
test_cmd += [
'--link', '%s:ansible.http.tests' % util_id,
'--link', '%s:sni1.ansible.http.tests' % util_id,
'--link', '%s:sni2.ansible.http.tests' % util_id,
'--link', '%s:fail.ansible.http.tests' % util_id,
'--env', 'HTTPTESTER=1',
]
test_id, _ = run_command(args, test_cmd + [test_image], capture=True)
if args.explain:
test_id = 'test_id'
else:
test_id = test_id.strip()
# write temporary files to /root since /tmp isn't ready immediately on container start
docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh')
run_command(args,
['docker', 'exec', test_id, '/bin/bash', '/root/docker.sh'])
docker_put(args, test_id, '/tmp/ansible.tgz', '/root/ansible.tgz')
run_command(args,
['docker', 'exec', test_id, 'mkdir', '/root/ansible'])
run_command(args,
['docker', 'exec', test_id, 'tar', 'oxzf', '/root/ansible.tgz', '--directory', '/root/ansible'])
try:
command = ['docker', 'exec']
if isinstance(args, ShellConfig):
command.append('-it')
run_command(args, command + [test_id] + cmd)
finally:
run_command(args,
['docker', 'exec', test_id,
'tar', 'czf', '/root/results.tgz', '--directory', '/root/ansible/test', 'results'])
docker_get(args, test_id, '/root/results.tgz', '/tmp/results.tgz')
run_command(args,
['tar', 'oxzf', '/tmp/results.tgz', '-C', 'test'])
finally:
if util_id:
run_command(args,
['docker', 'rm', '-f', util_id],
capture=True)
if test_id:
run_command(args,
['docker', 'rm', '-f', test_id],
capture=True)
def docker_put(args, container_id, src, dst):
"""
:type args: EnvironmentConfig
:type container_id: str
:type src: str
:type dst: str
"""
# avoid 'docker cp' due to a bug which causes 'docker rm' to fail
cmd = ['docker', 'exec', '-i', container_id, 'dd', 'of=%s' % dst, 'bs=%s' % BUFFER_SIZE]
with open(src, 'rb') as src_fd:
run_command(args, cmd, stdin=src_fd, capture=True)
def docker_get(args, container_id, src, dst):
"""
:type args: EnvironmentConfig
:type container_id: str
:type src: str
:type dst: str
"""
# avoid 'docker cp' due to a bug which causes 'docker rm' to fail
cmd = ['docker', 'exec', '-i', container_id, 'dd', 'if=%s' % src, 'bs=%s' % BUFFER_SIZE]
with open(dst, 'wb') as dst_fd:
run_command(args, cmd, stdout=dst_fd, capture=True)
def delegate_remote(args, exclude, require):
"""
:type args: EnvironmentConfig
:type exclude: list[str]
:type require: list[str]
"""
parts = args.remote.split('/', 1)
platform = parts[0]
version = parts[1]
core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage)
try:
core_ci.start()
core_ci.wait()
options = {
'--remote': 1,
}
cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require)
if isinstance(args, IntegrationConfig):
if not args.allow_destructive:
cmd.append('--allow-destructive')
manage = ManagePosixCI(core_ci)
manage.setup()
try:
manage.ssh(cmd)
finally:
manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results')
manage.download('/tmp/results', 'test')
finally:
pass
def generate_command(args, path, options, exclude, require):
"""
:type args: EnvironmentConfig
:type path: str
:type options: dict[str, int]
:type exclude: list[str]
:type require: list[str]
:return: list[str]
"""
options['--color'] = 1
cmd = [path]
cmd += list(filter_options(args, sys.argv[1:], options, exclude, require))
cmd += ['--color', 'yes' if args.color else 'no']
if args.requirements:
cmd += ['--requirements']
if isinstance(args, ShellConfig):
cmd = create_shell_command(cmd)
return cmd
def filter_options(args, argv, options, exclude, require):
"""
:type args: EnvironmentConfig
:type argv: list[str]
:type options: dict[str, int]
:type exclude: list[str]
:type require: list[str]
:rtype: collections.Iterable[str]
"""
options = options.copy()
options['--requirements'] = 0
if isinstance(args, TestConfig):
options.update({
'--changed': 0,
'--tracked': 0,
'--untracked': 0,
'--ignore-committed': 0,
'--ignore-staged': 0,
'--ignore-unstaged': 0,
'--changed-from': 1,
'--changed-path': 1,
})
remaining = 0
for arg in argv:
if not arg.startswith('-') and remaining:
remaining -= 1
continue
remaining = 0
parts = arg.split('=', 1)
key = parts[0]
if key in options:
remaining = options[key] - len(parts) + 1
continue
yield arg
for target in exclude:
yield '--exclude'
yield target
for target in require:
yield '--require'
yield target