community.general/test/runner/lib/delegation.py

"""Delegate test execution to another environment."""

from __future__ import absolute_import, print_function

import os
import sys
import time

import lib.pytar
import lib.thread

from lib.executor import (
    SUPPORTED_PYTHON_VERSIONS,
    EnvironmentConfig,
    IntegrationConfig,
    SubprocessError,
    ShellConfig,
    TestConfig,
    create_shell_command,
)

from lib.core_ci import (
    AnsibleCoreCI,
)

from lib.manage_ci import (
    ManagePosixCI,
)

from lib.util import (
    ApplicationError,
    run_command,
    display,
)

BUFFER_SIZE = 256 * 256


def delegate(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    if args.tox:
        delegate_tox(args, exclude, require)
        return True

    if args.docker:
        delegate_docker(args, exclude, require)
        return True

    if args.remote:
        delegate_remote(args, exclude, require)
        return True

    return False


def delegate_tox(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    if args.python:
        versions = args.python,

        if args.python not in SUPPORTED_PYTHON_VERSIONS:
            raise ApplicationError('tox does not support Python version %s' % args.python)
    else:
        versions = SUPPORTED_PYTHON_VERSIONS

    options = {
        '--tox': args.tox_args,
        '--tox-sitepackages': 0,
    }

    for version in versions:
        tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', '')]

        if args.tox_sitepackages:
            tox.append('--sitepackages')

        tox.append('--')

        cmd = generate_command(args, os.path.abspath('test/runner/test.py'), options, exclude, require)

        if not args.python:
            cmd += ['--python', version]

        run_command(args, tox + cmd)


def delegate_docker(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    util_image = args.docker_util
    test_image = args.docker
    privileged = args.docker_privileged

    if util_image:
        docker_pull(args, util_image)

    docker_pull(args, test_image)

    util_id = None
    test_id = None

    options = {
        '--docker': 1,
        '--docker-privileged': 0,
        '--docker-util': 1,
    }

    cmd = generate_command(args, '/root/ansible/test/runner/test.py', options, exclude, require)

    if isinstance(args, IntegrationConfig):
        if not args.allow_destructive:
            cmd.append('--allow-destructive')

    if not args.explain:
        lib.pytar.create_tarfile('/tmp/ansible.tgz', '.', lib.pytar.ignore)

    try:
        if util_image:
            util_id, _ = run_command(args, [
                'docker', 'run', '--detach',
                util_image,
            ], capture=True)

            if args.explain:
                util_id = 'util_id'
            else:
                util_id = util_id.strip()
        else:
            util_id = None

        test_cmd = [
            'docker', 'run', '--detach',
            '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro',
            '--privileged=%s' % str(privileged).lower(),
        ]

        if util_id:
            test_cmd += [
                '--link', '%s:ansible.http.tests' % util_id,
                '--link', '%s:sni1.ansible.http.tests' % util_id,
                '--link', '%s:sni2.ansible.http.tests' % util_id,
                '--link', '%s:fail.ansible.http.tests' % util_id,
                '--env', 'HTTPTESTER=1',
            ]

        test_id, _ = run_command(args, test_cmd + [test_image], capture=True)

        if args.explain:
            test_id = 'test_id'
        else:
            test_id = test_id.strip()

        # write temporary files to /root since /tmp isn't ready immediately on container start
        docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh')

        run_command(args,
                    ['docker', 'exec', test_id, '/bin/bash', '/root/docker.sh'])

        docker_put(args, test_id, '/tmp/ansible.tgz', '/root/ansible.tgz')

        run_command(args,
                    ['docker', 'exec', test_id, 'mkdir', '/root/ansible'])

        run_command(args,
                    ['docker', 'exec', test_id, 'tar', 'oxzf', '/root/ansible.tgz', '--directory', '/root/ansible'])

        try:
            command = ['docker', 'exec']

            if isinstance(args, ShellConfig):
                command.append('-it')

            run_command(args, command + [test_id] + cmd)
        finally:
            run_command(args,
                        ['docker', 'exec', test_id,
                         'tar', 'czf', '/root/results.tgz', '--directory', '/root/ansible/test', 'results'])

            docker_get(args, test_id, '/root/results.tgz', '/tmp/results.tgz')

            run_command(args,
                        ['tar', 'oxzf', '/tmp/results.tgz', '-C', 'test'])
    finally:
        if util_id:
            run_command(args,
                        ['docker', 'rm', '-f', util_id],
                        capture=True)

        if test_id:
            run_command(args,
                        ['docker', 'rm', '-f', test_id],
                        capture=True)


def docker_pull(args, image):
    """
    :type args: EnvironmentConfig
    :type image: str
    """
    if not args.docker_pull:
        display.warning('Skipping docker pull for "%s". Image may be out-of-date.' % image)
        return

    for _ in range(1, 10):
        try:
            run_command(args, ['docker', 'pull', image])
            return
        except SubprocessError:
            display.warning('Failed to pull docker image "%s". Waiting a few seconds before trying again.' % image)
            time.sleep(3)

    raise ApplicationError('Failed to pull docker image "%s".' % image)


def docker_put(args, container_id, src, dst):
    """
    :type args: EnvironmentConfig
    :type container_id: str
    :type src: str
    :type dst: str
    """
    # avoid 'docker cp' due to a bug which causes 'docker rm' to fail
    cmd = ['docker', 'exec', '-i', container_id, 'dd', 'of=%s' % dst, 'bs=%s' % BUFFER_SIZE]

    with open(src, 'rb') as src_fd:
        run_command(args, cmd, stdin=src_fd, capture=True)


def docker_get(args, container_id, src, dst):
    """
    :type args: EnvironmentConfig
    :type container_id: str
    :type src: str
    :type dst: str
    """
    # avoid 'docker cp' due to a bug which causes 'docker rm' to fail
    cmd = ['docker', 'exec', '-i', container_id, 'dd', 'if=%s' % src, 'bs=%s' % BUFFER_SIZE]

    with open(dst, 'wb') as dst_fd:
        run_command(args, cmd, stdout=dst_fd, capture=True)


def delegate_remote(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    parts = args.remote.split('/', 1)

    platform = parts[0]
    version = parts[1]

    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage)

    try:
        core_ci.start()
        core_ci.wait()

        options = {
            '--remote': 1,
        }

        cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require)

        if isinstance(args, IntegrationConfig):
            if not args.allow_destructive:
                cmd.append('--allow-destructive')

        manage = ManagePosixCI(core_ci)
        manage.setup()

        try:
            manage.ssh(cmd)
        finally:
            manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results')
            manage.download('/tmp/results', 'test')
    finally:
        pass


def generate_command(args, path, options, exclude, require):
    """
    :type args: EnvironmentConfig
    :type path: str
    :type options: dict[str, int]
    :type exclude: list[str]
    :type require: list[str]
    :return: list[str]
    """
    options['--color'] = 1

    cmd = [path]
    cmd += list(filter_options(args, sys.argv[1:], options, exclude, require))
    cmd += ['--color', 'yes' if args.color else 'no']

    if args.requirements:
        cmd += ['--requirements']

    if isinstance(args, ShellConfig):
        cmd = create_shell_command(cmd)

    return cmd


def filter_options(args, argv, options, exclude, require):
    """
    :type args: EnvironmentConfig
    :type argv: list[str]
    :type options: dict[str, int]
    :type exclude: list[str]
    :type require: list[str]
    :rtype: collections.Iterable[str]
    """
    options = options.copy()

    options['--requirements'] = 0

    if isinstance(args, TestConfig):
        options.update({
            '--changed': 0,
            '--tracked': 0,
            '--untracked': 0,
            '--ignore-committed': 0,
            '--ignore-staged': 0,
            '--ignore-unstaged': 0,
            '--changed-from': 1,
            '--changed-path': 1,
        })

    remaining = 0

    for arg in argv:
        if not arg.startswith('-') and remaining:
            remaining -= 1
            continue

        remaining = 0

        parts = arg.split('=', 1)
        key = parts[0]

        if key in options:
            remaining = options[key] - len(parts) + 1
            continue

        yield arg

    for target in exclude:
        yield '--exclude'
        yield target

    for target in require:
        yield '--require'
        yield target