From d8733a545583325626b5b69def0710bf198ced7a Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Sat, 7 Jan 2017 23:36:35 -0800 Subject: [PATCH] Complete initial network-integration support. --- .gitignore | 1 + test/runner/completion/network.txt | 4 ++ test/runner/lib/ansible_util.py | 1 + test/runner/lib/core_ci.py | 38 ++++++++++---- test/runner/lib/executor.py | 84 +++++++++++++++++++++++++++++- test/runner/lib/manage_ci.py | 35 +++++++++++++ test/runner/test.py | 17 ++++++ 7 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 test/runner/completion/network.txt diff --git a/.gitignore b/.gitignore index 4951ff3be3..0bd74f04d9 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ packaging/release/ansible_release /test/results/junit/*.xml /test/results/logs/*.log /test/integration/inventory.remote +/test/integration/inventory.networking /test/integration/inventory.winrm # old submodule dirs lib/ansible/modules/core diff --git a/test/runner/completion/network.txt b/test/runner/completion/network.txt new file mode 100644 index 0000000000..5b60f3409e --- /dev/null +++ b/test/runner/completion/network.txt @@ -0,0 +1,4 @@ +junos/vmx +junos/vsrx +vyos/1.0.5 +vyos/1.1.0 diff --git a/test/runner/lib/ansible_util.py b/test/runner/lib/ansible_util.py index ae74db7408..732c4cb76b 100644 --- a/test/runner/lib/ansible_util.py +++ b/test/runner/lib/ansible_util.py @@ -24,6 +24,7 @@ def ansible_environment(args): ANSIBLE_FORCE_COLOR='%s' % 'true' if args.color else 'false', ANSIBLE_DEPRECATION_WARNINGS='false', ANSIBLE_CONFIG='/dev/null', + ANSIBLE_HOST_KEY_CHECKING='false', PYTHONPATH=os.path.abspath('lib'), PAGER='/bin/cat', PATH=path, diff --git a/test/runner/lib/core_ci.py b/test/runner/lib/core_ci.py index 0580fe7f1f..bff8c251bc 100644 --- a/test/runner/lib/core_ci.py +++ b/test/runner/lib/core_ci.py @@ -45,17 +45,30 @@ class AnsibleCoreCI(object): self.instance_id = None self.name = name if name else '%s-%s' % (self.platform, self.version) - if self.platform == 'windows': - self.ssh_key = None + aws_platforms = ( + 'windows', + 'freebsd', + 'vyos', + 'junos', + ) + + osx_platforms = ( + 'osx', + ) + + if self.platform in aws_platforms: self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com' - self.port = 5986 - elif self.platform == 'freebsd': - self.ssh_key = SshKey(args) - self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com' - self.port = 22 - elif self.platform == 'osx': - self.ssh_key = SshKey(args) + + if self.platform == 'windows': + self.ssh_key = None + self.port = 5986 + else: + self.ssh_key = SshKey(args) + self.port = 22 + elif self.platform in osx_platforms: self.endpoint = 'https://osx.testing.ansible.com' + + self.ssh_key = SshKey(args) self.port = None else: raise ApplicationError('Unsupported platform: %s' % platform) @@ -213,8 +226,11 @@ class AnsibleCoreCI(object): verbosity=1) return - with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd: - winrm_config = winrm_config_fd.read() + if self.platform == 'windows': + with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd: + winrm_config = winrm_config_fd.read() + else: + winrm_config = None data = dict( config=dict( diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 9ef371d989..e18498cd95 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -25,6 +25,7 @@ from lib.core_ci import ( from lib.manage_ci import ( ManageWindowsCI, + ManageNetworkCI, ) from lib.util import ( @@ -182,9 +183,85 @@ def command_network_integration(args): :type args: NetworkIntegrationConfig """ internal_targets = command_integration_filter(args, walk_network_integration_targets()) + + if args.platform: + instances = [] # type: list [lib.thread.WrappedThread] + + for platform_version in args.platform: + platform, version = platform_version.split('/', 1) + instance = lib.thread.WrappedThread(functools.partial(network_run, args, platform, version)) + instance.daemon = True + instance.start() + instances.append(instance) + + install_command_requirements(args) + + while any(instance.is_alive() for instance in instances): + time.sleep(1) + + remotes = [instance.wait_for_result() for instance in instances] + inventory = network_inventory(remotes) + + if not args.explain: + with open('test/integration/inventory.networking', 'w') as inventory_fd: + display.info('>>> Inventory: %s\n%s' % (inventory_fd.name, inventory.strip()), verbosity=3) + inventory_fd.write(inventory) + else: + install_command_requirements(args) + command_integration_filtered(args, internal_targets) +def network_run(args, platform, version): + """ + :type args: NetworkIntegrationConfig + :type platform: str + :type version: str + :rtype: AnsibleCoreCI + """ + + core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage) + core_ci.start() + core_ci.wait() + + manage = ManageNetworkCI(core_ci) + manage.wait() + + return core_ci + + +def network_inventory(remotes): + """ + :type remotes: list[AnsibleCoreCI] + :rtype: str + """ + groups = dict([(remote.platform, []) for remote in remotes]) + + for remote in remotes: + groups[remote.platform].append( + '%s ansible_host=%s ansible_user=%s ansible_connection=network_cli ansible_ssh_private_key_file=%s' % ( + remote.name.replace('.', '_'), + remote.connection.hostname, + remote.connection.username, + remote.ssh_key.key, + ) + ) + + template = '' + + for group in groups: + hosts = '\n'.join(groups[group]) + + template += """ + [%s] + %s + """ % (group, hosts) + + inventory = textwrap.dedent(template) + + return inventory + + def command_windows_integration(args): """ :type args: WindowsIntegrationConfig @@ -210,6 +287,7 @@ def command_windows_integration(args): if not args.explain: with open('test/integration/inventory.winrm', 'w') as inventory_fd: + display.info('>>> Inventory: %s\n%s' % (inventory_fd.name, inventory.strip()), verbosity=3) inventory_fd.write(inventory) else: install_command_requirements(args) @@ -428,9 +506,11 @@ def command_integration_role(args, target, start_at_task): hosts = 'windows' gather_facts = False elif 'network/' in target.aliases: - inventory = 'inventory.network' + inventory = 'inventory.networking' hosts = target.name[:target.name.find('_')] gather_facts = False + if hosts == 'net': + hosts = 'all' else: inventory = 'inventory' hosts = 'testhost' @@ -1215,6 +1295,8 @@ class NetworkIntegrationConfig(IntegrationConfig): """ super(NetworkIntegrationConfig, self).__init__(args, 'network-integration') + self.platform = args.platform # type list [str] + class UnitsConfig(TestConfig): """Configuration for the units command.""" diff --git a/test/runner/lib/manage_ci.py b/test/runner/lib/manage_ci.py index 85839cd644..a2d952c64d 100644 --- a/test/runner/lib/manage_ci.py +++ b/test/runner/lib/manage_ci.py @@ -59,6 +59,41 @@ class ManageWindowsCI(object): (self.core_ci.platform, self.core_ci.version, self.core_ci.instance_id)) +class ManageNetworkCI(object): + """Manage access to a network instance provided by Ansible Core CI.""" + def __init__(self, core_ci): + """ + :type core_ci: AnsibleCoreCI + """ + self.core_ci = core_ci + + def wait(self): + """Wait for instance to respond to ansible ping.""" + extra_vars = [ + 'ansible_host=%s' % self.core_ci.connection.hostname, + 'ansible_user=%s' % self.core_ci.connection.username, + 'ansible_port=%s' % self.core_ci.connection.port, + 'ansible_connection=network_cli', + 'ansible_ssh_private_key_file=%s' % self.core_ci.ssh_key.key, + ] + + name = '%s-%s' % (self.core_ci.platform, self.core_ci.version.replace('.', '_')) + + env = ansible_environment(self.core_ci.args) + cmd = ['ansible', '-m', 'net_command', '-a', '?', '-i', '%s,' % name, name, '-e', ' '.join(extra_vars)] + + for _ in range(1, 90): + try: + run_command(self.core_ci.args, cmd, env=env) + return + except SubprocessError: + sleep(10) + continue + + raise ApplicationError('Timeout waiting for %s/%s instance %s.' % + (self.core_ci.platform, self.core_ci.version, self.core_ci.instance_id)) + + class ManagePosixCI(object): """Manage access to a POSIX instance provided by Ansible Core CI.""" def __init__(self, core_ci): diff --git a/test/runner/test.py b/test/runner/test.py index 9cce048c15..e89b31ddfb 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -194,6 +194,11 @@ def parse_args(): targets=walk_network_integration_targets, config=NetworkIntegrationConfig) + network_integration.add_argument('--platform', + metavar='PLATFORM', + action='append', + help='network platform/version').completer = complete_network_platform + windows_integration = subparsers.add_parser('windows-integration', parents=[integration], help='windows integration tests') @@ -503,5 +508,17 @@ def complete_windows(prefix, parsed_args, **_): return [i for i in images if i.startswith(prefix) and (not parsed_args.windows or i not in parsed_args.windows)] +def complete_network_platform(prefix, parsed_args, **_): + """ + :type prefix: unicode + :type parsed_args: any + :rtype: list[str] + """ + with open('test/runner/completion/network.txt', 'r') as completion_fd: + images = completion_fd.read().splitlines() + + return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)] + + if __name__ == '__main__': main()