Complete initial network-integration support.

.gitignore

@@ -73,6 +73,7 @@ packaging/release/ansible_release
 /test/results/junit/*.xml
 /test/results/logs/*.log
 /test/integration/inventory.remote
+/test/integration/inventory.networking
 /test/integration/inventory.winrm
 # old submodule dirs
 lib/ansible/modules/core

test/runner/completion/network.txt

@@ -0,0 +1,4 @@
+junos/vmx
+junos/vsrx
+vyos/1.0.5
+vyos/1.1.0

test/runner/lib/ansible_util.py

@@ -24,6 +24,7 @@ def ansible_environment(args):
         ANSIBLE_FORCE_COLOR='%s' % 'true' if args.color else 'false',
         ANSIBLE_DEPRECATION_WARNINGS='false',
         ANSIBLE_CONFIG='/dev/null',
+        ANSIBLE_HOST_KEY_CHECKING='false',
         PYTHONPATH=os.path.abspath('lib'),
         PAGER='/bin/cat',
         PATH=path,

test/runner/lib/core_ci.py

@@ -45,17 +45,30 @@ class AnsibleCoreCI(object):
         self.instance_id = None
         self.name = name if name else '%s-%s' % (self.platform, self.version)
 
+        aws_platforms = (
+            'windows',
+            'freebsd',
+            'vyos',
+            'junos',
+        )
+
+        osx_platforms = (
+            'osx',
+        )
+
+        if self.platform in aws_platforms:
+            self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
+
             if self.platform == 'windows':
                 self.ssh_key = None
-            self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
                 self.port = 5986
-        elif self.platform == 'freebsd':
+            else:
                 self.ssh_key = SshKey(args)
-            self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
                 self.port = 22
-        elif self.platform == 'osx':
+        elif self.platform in osx_platforms:
-            self.ssh_key = SshKey(args)
             self.endpoint = 'https://osx.testing.ansible.com'
+
+            self.ssh_key = SshKey(args)
             self.port = None
         else:
             raise ApplicationError('Unsupported platform: %s' % platform)
@@ -213,8 +226,11 @@ class AnsibleCoreCI(object):
                          verbosity=1)
             return
 
+        if self.platform == 'windows':
             with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd:
                 winrm_config = winrm_config_fd.read()
+        else:
+            winrm_config = None
 
         data = dict(
             config=dict(

test/runner/lib/executor.py

@@ -25,6 +25,7 @@ from lib.core_ci import (
 
 from lib.manage_ci import (
     ManageWindowsCI,
+    ManageNetworkCI,
 )
 
 from lib.util import (
@@ -182,9 +183,85 @@ def command_network_integration(args):
     :type args: NetworkIntegrationConfig
     """
     internal_targets = command_integration_filter(args, walk_network_integration_targets())
+
+    if args.platform:
+        instances = []  # type: list [lib.thread.WrappedThread]
+
+        for platform_version in args.platform:
+            platform, version = platform_version.split('/', 1)
+            instance = lib.thread.WrappedThread(functools.partial(network_run, args, platform, version))
+            instance.daemon = True
+            instance.start()
+            instances.append(instance)
+
+        install_command_requirements(args)
+
+        while any(instance.is_alive() for instance in instances):
+            time.sleep(1)
+
+        remotes = [instance.wait_for_result() for instance in instances]
+        inventory = network_inventory(remotes)
+
+        if not args.explain:
+            with open('test/integration/inventory.networking', 'w') as inventory_fd:
+                display.info('>>> Inventory: %s\n%s' % (inventory_fd.name, inventory.strip()), verbosity=3)
+                inventory_fd.write(inventory)
+    else:
+        install_command_requirements(args)
+
     command_integration_filtered(args, internal_targets)
 
 
+def network_run(args, platform, version):
+    """
+    :type args: NetworkIntegrationConfig
+    :type platform: str
+    :type version: str
+    :rtype: AnsibleCoreCI
+    """
+
+    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage)
+    core_ci.start()
+    core_ci.wait()
+
+    manage = ManageNetworkCI(core_ci)
+    manage.wait()
+
+    return core_ci
+
+
+def network_inventory(remotes):
+    """
+    :type remotes: list[AnsibleCoreCI]
+    :rtype: str
+    """
+    groups = dict([(remote.platform, []) for remote in remotes])
+
+    for remote in remotes:
+        groups[remote.platform].append(
+            '%s ansible_host=%s ansible_user=%s ansible_connection=network_cli ansible_ssh_private_key_file=%s' % (
+                remote.name.replace('.', '_'),
+                remote.connection.hostname,
+                remote.connection.username,
+                remote.ssh_key.key,
+            )
+        )
+
+    template = ''
+
+    for group in groups:
+        hosts = '\n'.join(groups[group])
+
+        template += """
+        [%s]
+        %s
+        """ % (group, hosts)
+
+    inventory = textwrap.dedent(template)
+
+    return inventory
+
+
 def command_windows_integration(args):
     """
     :type args: WindowsIntegrationConfig
@@ -210,6 +287,7 @@ def command_windows_integration(args):
 
         if not args.explain:
             with open('test/integration/inventory.winrm', 'w') as inventory_fd:
+                display.info('>>> Inventory: %s\n%s' % (inventory_fd.name, inventory.strip()), verbosity=3)
                 inventory_fd.write(inventory)
     else:
         install_command_requirements(args)
@@ -428,9 +506,11 @@ def command_integration_role(args, target, start_at_task):
         hosts = 'windows'
         gather_facts = False
     elif 'network/' in target.aliases:
-        inventory = 'inventory.network'
+        inventory = 'inventory.networking'
         hosts = target.name[:target.name.find('_')]
         gather_facts = False
+        if hosts == 'net':
+            hosts = 'all'
     else:
         inventory = 'inventory'
         hosts = 'testhost'
@@ -1215,6 +1295,8 @@ class NetworkIntegrationConfig(IntegrationConfig):
         """
         super(NetworkIntegrationConfig, self).__init__(args, 'network-integration')
 
+        self.platform = args.platform  # type list [str]
+
 
 class UnitsConfig(TestConfig):
     """Configuration for the units command."""

test/runner/lib/manage_ci.py

@@ -59,6 +59,41 @@ class ManageWindowsCI(object):
                                (self.core_ci.platform, self.core_ci.version, self.core_ci.instance_id))
 
 
+class ManageNetworkCI(object):
+    """Manage access to a network instance provided by Ansible Core CI."""
+    def __init__(self, core_ci):
+        """
+        :type core_ci: AnsibleCoreCI
+        """
+        self.core_ci = core_ci
+
+    def wait(self):
+        """Wait for instance to respond to ansible ping."""
+        extra_vars = [
+            'ansible_host=%s' % self.core_ci.connection.hostname,
+            'ansible_user=%s' % self.core_ci.connection.username,
+            'ansible_port=%s' % self.core_ci.connection.port,
+            'ansible_connection=network_cli',
+            'ansible_ssh_private_key_file=%s' % self.core_ci.ssh_key.key,
+        ]
+
+        name = '%s-%s' % (self.core_ci.platform, self.core_ci.version.replace('.', '_'))
+
+        env = ansible_environment(self.core_ci.args)
+        cmd = ['ansible', '-m', 'net_command', '-a', '?', '-i', '%s,' % name, name, '-e', ' '.join(extra_vars)]
+
+        for _ in range(1, 90):
+            try:
+                run_command(self.core_ci.args, cmd, env=env)
+                return
+            except SubprocessError:
+                sleep(10)
+                continue
+
+        raise ApplicationError('Timeout waiting for %s/%s instance %s.' %
+                               (self.core_ci.platform, self.core_ci.version, self.core_ci.instance_id))
+
+
 class ManagePosixCI(object):
     """Manage access to a POSIX instance provided by Ansible Core CI."""
     def __init__(self, core_ci):

test/runner/test.py

@@ -194,6 +194,11 @@ def parse_args():
                                      targets=walk_network_integration_targets,
                                      config=NetworkIntegrationConfig)
 
+    network_integration.add_argument('--platform',
+                                     metavar='PLATFORM',
+                                     action='append',
+                                     help='network platform/version').completer = complete_network_platform
+
     windows_integration = subparsers.add_parser('windows-integration',
                                                 parents=[integration],
                                                 help='windows integration tests')
@@ -503,5 +508,17 @@ def complete_windows(prefix, parsed_args, **_):
     return [i for i in images if i.startswith(prefix) and (not parsed_args.windows or i not in parsed_args.windows)]
 
 
+def complete_network_platform(prefix, parsed_args, **_):
+    """
+    :type prefix: unicode
+    :type parsed_args: any
+    :rtype: list[str]
+    """
+    with open('test/runner/completion/network.txt', 'r') as completion_fd:
+        images = completion_fd.read().splitlines()
+
+    return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)]
+
+
 if __name__ == '__main__':
     main()