From c1f9efabf474b541644b7019e8c4e9c6e700e475 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 9 May 2018 09:24:39 -0700 Subject: [PATCH] Overhaul httptester support in ansible-test. (#39892) - Works with the --remote option. - Can be disabled with the --disable-httptester option. - Change image with the --httptester option. - Only load and run httptester for targets that require it. --- test/integration/targets/get_url/aliases | 1 + test/integration/targets/lookups/aliases | 1 + .../targets/prepare_http_tests/tasks/main.yml | 22 +++ test/integration/targets/uri/aliases | 1 + test/runner/lib/config.py | 4 +- test/runner/lib/delegation.py | 108 +++++++----- test/runner/lib/docker_util.py | 19 +++ test/runner/lib/executor.py | 158 +++++++++++++++++- test/runner/lib/util.py | 14 ++ test/runner/test.py | 32 +++- 10 files changed, 313 insertions(+), 47 deletions(-) diff --git a/test/integration/targets/get_url/aliases b/test/integration/targets/get_url/aliases index 8e7d715f9c..632a671d27 100644 --- a/test/integration/targets/get_url/aliases +++ b/test/integration/targets/get_url/aliases @@ -1,2 +1,3 @@ destructive posix/ci/group1 +needs/httptester diff --git a/test/integration/targets/lookups/aliases b/test/integration/targets/lookups/aliases index 7af8b7f05b..58ddd39dc7 100644 --- a/test/integration/targets/lookups/aliases +++ b/test/integration/targets/lookups/aliases @@ -1 +1,2 @@ posix/ci/group2 +needs/httptester diff --git a/test/integration/targets/prepare_http_tests/tasks/main.yml b/test/integration/targets/prepare_http_tests/tasks/main.yml index c98c783df5..c8c7fa99e5 100644 --- a/test/integration/targets/prepare_http_tests/tasks/main.yml +++ b/test/integration/targets/prepare_http_tests/tasks/main.yml @@ -46,4 +46,26 @@ command: update-ca-certificates when: ansible_os_family == 'Debian' or ansible_os_family == 'Suse' + - name: FreeBSD - Retrieve test cacert + get_url: + url: "http://ansible.http.tests/cacert.pem" + dest: "/tmp/ansible.pem" + when: ansible_os_family == 'FreeBSD' + + - name: FreeBSD - Add cacert to root certificate store + blockinfile: + path: "/etc/ssl/cert.pem" + block: "{{ lookup('file', '/tmp/ansible.pem') }}" + when: ansible_os_family == 'FreeBSD' + + - name: MacOS - Retrieve test cacert + get_url: + url: "http://ansible.http.tests/cacert.pem" + dest: "/usr/local/etc/openssl/certs/ansible.pem" + when: ansible_os_family == 'Darwin' + + - name: MacOS - Update ca certificates + command: /usr/local/opt/openssl/bin/c_rehash + when: ansible_os_family == 'Darwin' + when: has_httptester|bool diff --git a/test/integration/targets/uri/aliases b/test/integration/targets/uri/aliases index 8e7d715f9c..632a671d27 100644 --- a/test/integration/targets/uri/aliases +++ b/test/integration/targets/uri/aliases @@ -1,2 +1,3 @@ destructive posix/ci/group1 +needs/httptester diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index 90480c2a2c..ba9e7442d2 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -43,7 +43,6 @@ class EnvironmentConfig(CommonConfig): self.remote = args.remote # type: str self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False # type: bool - self.docker_util = docker_qualify_image(args.docker_util if 'docker_util' in args else '') # type: str self.docker_pull = args.docker_pull if 'docker_pull' in args else False # type: bool self.docker_keep_git = args.docker_keep_git if 'docker_keep_git' in args else False # type: bool self.docker_memory = args.docker_memory if 'docker_memory' in args else None @@ -70,6 +69,9 @@ class EnvironmentConfig(CommonConfig): if self.delegate: self.requirements = True + self.inject_httptester = args.inject_httptester if 'inject_httptester' in args else False # type: bool + self.httptester = docker_qualify_image(args.httptester if 'httptester' in args else '') # type: str + @property def python_executable(self): """ diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index 1bb6721977..d086e991e3 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -12,7 +12,10 @@ import lib.thread from lib.executor import ( SUPPORTED_PYTHON_VERSIONS, + HTTPTESTER_HOSTS, create_shell_command, + run_httptester, + start_httptester, ) from lib.config import ( @@ -37,6 +40,7 @@ from lib.util import ( run_command, common_environment, pass_vars, + display, ) from lib.docker_util import ( @@ -46,18 +50,24 @@ from lib.docker_util import ( docker_put, docker_rm, docker_run, + docker_available, ) from lib.cloud import ( get_cloud_providers, ) +from lib.target import ( + IntegrationTarget, +) -def delegate(args, exclude, require): + +def delegate(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] + :type integration_targets: tuple[IntegrationTarget] :rtype: bool """ if isinstance(args, TestConfig): @@ -66,40 +76,42 @@ def delegate(args, exclude, require): args.metadata.to_file(args.metadata_path) try: - return delegate_command(args, exclude, require) + return delegate_command(args, exclude, require, integration_targets) finally: args.metadata_path = None else: - return delegate_command(args, exclude, require) + return delegate_command(args, exclude, require, integration_targets) -def delegate_command(args, exclude, require): +def delegate_command(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] + :type integration_targets: tuple[IntegrationTarget] :rtype: bool """ if args.tox: - delegate_tox(args, exclude, require) + delegate_tox(args, exclude, require, integration_targets) return True if args.docker: - delegate_docker(args, exclude, require) + delegate_docker(args, exclude, require, integration_targets) return True if args.remote: - delegate_remote(args, exclude, require) + delegate_remote(args, exclude, require, integration_targets) return True return False -def delegate_tox(args, exclude, require): +def delegate_tox(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] + :type integration_targets: tuple[IntegrationTarget] """ if args.python: versions = args.python_version, @@ -109,6 +121,12 @@ def delegate_tox(args, exclude, require): else: versions = SUPPORTED_PYTHON_VERSIONS + if args.httptester: + needs_httptester = sorted(target.name for target in integration_targets if 'needs/httptester/' in target.aliases) + + if needs_httptester: + display.warning('Use --docker or --remote to enable httptester for tests marked "needs/httptester": %s' % ', '.join(needs_httptester)) + options = { '--tox': args.tox_args, '--tox-sitepackages': 0, @@ -145,22 +163,27 @@ def delegate_tox(args, exclude, require): run_command(args, tox + cmd, env=env) -def delegate_docker(args, exclude, require): +def delegate_docker(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] + :type integration_targets: tuple[IntegrationTarget] """ - util_image = args.docker_util test_image = args.docker privileged = args.docker_privileged - if util_image: - docker_pull(args, util_image) + if isinstance(args, ShellConfig): + use_httptester = args.httptester + else: + use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets) + + if use_httptester: + docker_pull(args, args.httptester) docker_pull(args, test_image) - util_id = None + httptester_id = None test_id = None options = { @@ -196,19 +219,10 @@ def delegate_docker(args, exclude, require): lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter) - if util_image: - util_options = [ - '--detach', - ] - - util_id, _ = docker_run(args, util_image, options=util_options) - - if args.explain: - util_id = 'util_id' - else: - util_id = util_id.strip() + if use_httptester: + httptester_id = run_httptester(args) else: - util_id = None + httptester_id = None test_options = [ '--detach', @@ -227,14 +241,11 @@ def delegate_docker(args, exclude, require): if os.path.exists(docker_socket): test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)] - if util_id: - test_options += [ - '--link', '%s:ansible.http.tests' % util_id, - '--link', '%s:sni1.ansible.http.tests' % util_id, - '--link', '%s:sni2.ansible.http.tests' % util_id, - '--link', '%s:fail.ansible.http.tests' % util_id, - '--env', 'HTTPTESTER=1', - ] + if httptester_id: + test_options += ['--env', 'HTTPTESTER=1'] + + for host in HTTPTESTER_HOSTS: + test_options += ['--link', '%s:%s' % (httptester_id, host)] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) @@ -268,18 +279,19 @@ def delegate_docker(args, exclude, require): docker_get(args, test_id, '/root/results.tgz', local_result_fd.name) run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test']) finally: - if util_id: - docker_rm(args, util_id) + if httptester_id: + docker_rm(args, httptester_id) if test_id: docker_rm(args, test_id) -def delegate_remote(args, exclude, require): +def delegate_remote(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] + :type integration_targets: tuple[IntegrationTarget] """ parts = args.remote.split('/', 1) @@ -289,8 +301,24 @@ def delegate_remote(args, exclude, require): core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) success = False + if isinstance(args, ShellConfig): + use_httptester = args.httptester + else: + use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets) + + if use_httptester and not docker_available(): + display.warning('Assuming --disable-httptester since `docker` is not available.') + use_httptester = False + + httptester_id = None + ssh_options = [] + try: core_ci.start() + + if use_httptester: + httptester_id, ssh_options = start_httptester(args) + core_ci.wait() options = { @@ -299,6 +327,9 @@ def delegate_remote(args, exclude, require): cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require) + if httptester_id: + cmd += ['--inject-httptester'] + if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: cmd += ['--coverage-label', 'remote-%s-%s' % (platform, version)] @@ -314,8 +345,6 @@ def delegate_remote(args, exclude, require): manage = ManagePosixCI(core_ci) manage.setup() - ssh_options = [] - if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) @@ -332,6 +361,9 @@ def delegate_remote(args, exclude, require): if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success): core_ci.stop() + if httptester_id: + docker_rm(args, httptester_id) + def generate_command(args, path, options, exclude, require): """ diff --git a/test/runner/lib/docker_util.py b/test/runner/lib/docker_util.py index d150bef828..691d73d45c 100644 --- a/test/runner/lib/docker_util.py +++ b/test/runner/lib/docker_util.py @@ -15,6 +15,7 @@ from lib.util import ( run_command, common_environment, display, + find_executable, ) from lib.config import ( @@ -24,6 +25,13 @@ from lib.config import ( BUFFER_SIZE = 256 * 256 +def docker_available(): + """ + :rtype: bool + """ + return find_executable('docker', required=False) + + def get_docker_container_id(): """ :rtype: str | None @@ -48,6 +56,17 @@ def get_docker_container_id(): raise ApplicationError('Found multiple container_id candidates: %s\n%s' % (sorted(container_ids), contents)) +def get_docker_container_ip(args, container_id): + """ + :type args: EnvironmentConfig + :type container_id: str + :rtype: str + """ + results = docker_inspect(args, container_id) + ipaddress = results[0]['NetworkSettings']['IPAddress'] + return ipaddress + + def docker_pull(args, image): """ :type args: EnvironmentConfig diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index f9b9d45bcc..7a01081316 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -48,6 +48,14 @@ from lib.util import ( find_executable, raw_command, get_coverage_path, + get_available_port, +) + +from lib.docker_util import ( + docker_pull, + docker_run, + get_docker_container_id, + get_docker_container_ip, ) from lib.ansible_util import ( @@ -100,6 +108,12 @@ SUPPORTED_PYTHON_VERSIONS = ( '3.7', ) +HTTPTESTER_HOSTS = ( + 'ansible.http.tests', + 'sni1.ansible.http.tests', + 'fail.ansible.http.tests', +) + def check_startup(): """Checks to perform at startup before running commands.""" @@ -277,6 +291,9 @@ def command_shell(args): install_command_requirements(args) + if args.inject_httptester: + inject_httptester(args) + cmd = create_shell_command(['bash', '-i']) run_command(args, cmd) @@ -649,7 +666,7 @@ def command_integration_filter(args, targets, init_callback=None): cloud_init(args, internal_targets) if args.delegate: - raise Delegate(require=changes, exclude=exclude) + raise Delegate(require=changes, exclude=exclude, integration_targets=internal_targets) install_command_requirements(args) @@ -697,6 +714,9 @@ def command_integration_filtered(args, targets, all_targets): display.warning('SSH service not responding. Waiting %d second(s) before checking again.' % seconds) time.sleep(seconds) + if args.inject_httptester: + inject_httptester(args) + start_at_task = args.start_at_task results = {} @@ -815,6 +835,133 @@ def command_integration_filtered(args, targets, all_targets): len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed))) +def start_httptester(args): + """ + :type args: EnvironmentConfig + :rtype: str, list[str] + """ + + # map ports from remote -> localhost -> container + # passing through localhost is only used when ansible-test is not already running inside a docker container + ports = [ + dict( + remote=8080, + container=80, + ), + dict( + remote=8443, + container=443, + ), + ] + + container_id = get_docker_container_id() + + if container_id: + display.info('Running in docker container: %s' % container_id, verbosity=1) + else: + for item in ports: + item['localhost'] = get_available_port() + + docker_pull(args, args.httptester) + + httptester_id = run_httptester(args, dict((port['localhost'], port['container']) for port in ports if 'localhost' in port)) + + if container_id: + container_host = get_docker_container_ip(args, httptester_id) + display.info('Found httptester container address: %s' % container_host, verbosity=1) + else: + container_host = 'localhost' + + ssh_options = [] + + for port in ports: + ssh_options += ['-R', '%d:%s:%d' % (port['remote'], container_host, port.get('localhost', port['container']))] + + return httptester_id, ssh_options + + +def run_httptester(args, ports=None): + """ + :type args: EnvironmentConfig + :type ports: dict[int, int] | None + :rtype: str + """ + options = [ + '--detach', + ] + + if ports: + for localhost_port, container_port in ports.items(): + options += ['-p', '%d:%d' % (localhost_port, container_port)] + + httptester_id, _ = docker_run(args, args.httptester, options=options) + + if args.explain: + httptester_id = 'httptester_id' + else: + httptester_id = httptester_id.strip() + + return httptester_id + + +def inject_httptester(args): + """ + :type args: CommonConfig + """ + comment = ' # ansible-test httptester\n' + append_lines = ['127.0.0.1 %s%s' % (host, comment) for host in HTTPTESTER_HOSTS] + + with open('/etc/hosts', 'r+') as hosts_fd: + original_lines = hosts_fd.readlines() + + if not any(line.endswith(comment) for line in original_lines): + hosts_fd.writelines(append_lines) + + # determine which forwarding mechanism to use + pfctl = find_executable('pfctl', required=False) + iptables = find_executable('iptables', required=False) + + if pfctl: + kldload = find_executable('kldload', required=False) + + if kldload: + try: + run_command(args, ['kldload', 'pf'], capture=True) + except SubprocessError: + pass # already loaded + + rules = ''' +rdr pass inet proto tcp from any to any port 80 -> 127.0.0.1 port 8080 +rdr pass inet proto tcp from any to any port 443 -> 127.0.0.1 port 8443 +''' + cmd = ['pfctl', '-ef', '-'] + + try: + run_command(args, cmd, capture=True, data=rules) + except SubprocessError: + pass # non-zero exit status on success + + elif iptables: + ports = [ + (80, 8080), + (443, 8443), + ] + + for src, dst in ports: + rule = ['-o', 'lo', '-p', 'tcp', '--dport', str(src), '-j', 'REDIRECT', '--to-port', str(dst)] + + try: + # check for existing rule + cmd = ['iptables', '-t', 'nat', '-C', 'OUTPUT'] + rule + run_command(args, cmd, capture=True) + except SubprocessError: + # append rule when it does not exist + cmd = ['iptables', '-t', 'nat', '-A', 'OUTPUT'] + rule + run_command(args, cmd, capture=True) + else: + raise ApplicationError('No supported port forwarding mechanism detected.') + + def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, always): """ :type args: IntegrationConfig @@ -852,6 +999,11 @@ def integration_environment(args, target, cmd): """ env = ansible_environment(args) + if args.inject_httptester: + env.update(dict( + HTTPTESTER='1', + )) + integration = dict( JUNIT_OUTPUT_DIR=os.path.abspath('test/results/junit'), ANSIBLE_CALLBACK_WHITELIST='junit', @@ -1464,15 +1616,17 @@ class NoTestsForChanges(ApplicationWarning): class Delegate(Exception): """Trigger command delegation.""" - def __init__(self, exclude=None, require=None): + def __init__(self, exclude=None, require=None, integration_targets=None): """ :type exclude: list[str] | None :type require: list[str] | None + :type integration_targets: tuple[IntegrationTarget] | None """ super(Delegate, self).__init__() self.exclude = exclude or [] self.require = require or [] + self.integration_targets = integration_targets or tuple() class AllTargetsSkipped(ApplicationWarning): diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index cf80269ad5..2c60da6fe3 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, print_function import atexit +import contextlib import errno import filecmp import fcntl @@ -14,6 +15,7 @@ import pkgutil import random import re import shutil +import socket import stat import string import subprocess @@ -720,6 +722,18 @@ def parse_to_dict(pattern, value): return match.groupdict() +def get_available_port(): + """ + :rtype: int + """ + # this relies on the kernel not reusing previously assigned ports immediately + socket_fd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + with contextlib.closing(socket_fd): + socket_fd.bind(('', 0)) + return socket_fd.getsockname()[1] + + def get_subclasses(class_type): """ :type class_type: type diff --git a/test/runner/test.py b/test/runner/test.py index f0f79d2634..aacc38f1bd 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -88,7 +88,7 @@ def main(): try: args.func(config) except Delegate as ex: - delegate(config, ex.exclude, ex.require) + delegate(config, ex.exclude, ex.require, ex.integration_targets) display.review_warnings() except ApplicationWarning as ex: @@ -278,6 +278,7 @@ def parse_args(): config=PosixIntegrationConfig) add_extra_docker_options(posix_integration) + add_httptester_options(posix_integration, argparse) network_integration = subparsers.add_parser('network-integration', parents=[integration], @@ -380,6 +381,7 @@ def parse_args(): add_environments(shell, tox_version=True) add_extra_docker_options(shell) + add_httptester_options(shell, argparse) coverage_common = argparse.ArgumentParser(add_help=False, parents=[common]) @@ -606,6 +608,29 @@ def add_extra_coverage_options(parser): help='generate empty report of all python source files') +def add_httptester_options(parser, argparse): + """ + :type parser: argparse.ArgumentParser + :type argparse: argparse + """ + group = parser.add_mutually_exclusive_group() + + group.add_argument('--httptester', + metavar='IMAGE', + default='quay.io/ansible/http-test-container:1.0.0', + help='docker image to use for the httptester container') + + group.add_argument('--disable-httptester', + dest='httptester', + action='store_const', + const='', + help='do not use the httptester container') + + parser.add_argument('--inject-httptester', + action='store_true', + help=argparse.SUPPRESS) # internal use only + + def add_extra_docker_options(parser, integration=True): """ :type parser: argparse.ArgumentParser @@ -625,11 +650,6 @@ def add_extra_docker_options(parser, integration=True): if not integration: return - docker.add_argument('--docker-util', - metavar='IMAGE', - default='quay.io/ansible/http-test-container:1.0.0', - help='docker utility image to provide test services') - docker.add_argument('--docker-privileged', action='store_true', help='run docker container in privileged mode')