Add Azure support to ansible-test. (#28092)

* Initial Azure cloud testing support.
* Add missing Azure requirements.
* Fix test handling of setup and requirements.
* Update Azure cloud plugin.
* Add setup_azure role for integration tests.
* Update minimal Azure integration test sample.

packaging/requirements/requirements-azure.txt

@@ -1,3 +1,5 @@
+packaging
+requests[security]
 azure-mgmt-compute>=2.0.0,<3
 azure-mgmt-network>=1.3.0,<2
 azure-mgmt-storage>=1.2.0,<2

test/integration/cloud-config-azure.yml.template

@@ -0,0 +1,31 @@
+# This is the configuration template for ansible-test Azure integration tests.
+#
+# You do not need this template if you are:
+#
+# 1) Running integration tests without using ansible-test.
+# 2) Using the automatically provisioned Azure credentials in ansible-test.
+#
+# If you do not want to use the automatically provisioned temporary Azure credentials,
+# fill in the values below and save this file without the .template extension.
+# This will cause ansible-test to use the given configuration instead of temporary credentials.
+#
+# NOTE: Automatic provisioning of Azure credentials requires one of:
+#       1) ansible-core-ci API key in ~/.ansible-core-ci.key
+#       2) Sherlock URL (including API key) in ~/.ansible-sherlock-ci.cfg
+
+# Provide either Service Principal or Active Directory credentials below.
+
+# Service Principal
+AZURE_CLIENT_ID=
+AZURE_SECRET=
+AZURE_SUBSCRIPTION_ID=
+AZURE_TENANT=
+
+# Active Directory
+AZURE_AD_USER=
+AZURE_PASSWORD=
+AZURE_SUBSCRIPTION_ID=
+
+# Resource Groups
+RESOURCE_GROUP=
+RESOURCE_GROUP_SECONDARY=

test/integration/inventory

@@ -50,3 +50,6 @@ overridden_in_parent=2000
 
 [amazon]
 localhost ansible_ssh_host=127.0.0.1 ansible_connection=local
+
+[azure]
+localhost ansible_ssh_host=127.0.0.1 ansible_connection=local

test/integration/targets/azure_rm_virtualnetwork/aliases

@@ -0,0 +1,2 @@
+cloud/azure
+destructive

test/integration/targets/azure_rm_virtualnetwork/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - setup_azure

test/integration/targets/azure_rm_virtualnetwork/tasks/main.yml

@@ -0,0 +1,7 @@
+- block:
+  - name: create a virtual network
+    azure_rm_virtualnetwork:
+      resource_group: '{{ resource_group }}'
+      name: test
+      address_prefixes_cidr:
+        - "10.1.0.0/16"

test/integration/targets/setup_azure/tasks/main.yml

@@ -0,0 +1,2 @@
+- pip:
+    requirements: '{{ role_path }}/../../../../packaging/requirements/requirements-azure.txt'

test/runner/lib/classification.py

@@ -347,10 +347,21 @@ class PathMapper(object):
 
         if path.startswith('packaging/'):
             if path.startswith('packaging/requirements/'):
+                if name.startswith('requirements-') and ext == '.txt':
+                    component = name.split('-', 1)[1]
+
+                    candidates = (
+                        'cloud/%s/' % component,
+                    )
+
+                    for candidate in candidates:
+                        if candidate in self.integration_targets_by_alias:
                             return {
-                    'integration': 'ansible',
+                                'integration': candidate,
                             }
 
+                return all_tests(self.args)  # broad impact, run all tests
+
             return minimal
 
         if path.startswith('test/compile/'):
@@ -467,9 +478,7 @@ class PathMapper(object):
                 return all_tests(self.args)  # test infrastructure, run all tests
 
             if path == 'setup.py':
-                return {
+                return all_tests(self.args)  # broad impact, run all tests
-                    'integration': 'ansible',
-                }
 
             if path == '.yamllint':
                 return {

test/runner/lib/cloud/azure.py

@@ -0,0 +1,194 @@
+"""Azure plugin for integration tests."""
+from __future__ import absolute_import, print_function
+
+import os
+
+from lib.util import (
+    ApplicationError,
+    display,
+    is_shippable,
+)
+
+from lib.cloud import (
+    CloudProvider,
+    CloudEnvironment,
+)
+
+from lib.http import (
+    HttpClient,
+    urlparse,
+    urlunparse,
+    parse_qs,
+)
+
+from lib.core_ci import (
+    AnsibleCoreCI,
+)
+
+
+class AzureCloudProvider(CloudProvider):
+    """Azure cloud provider plugin. Sets up cloud resources before delegation."""
+    SHERLOCK_CONFIG_PATH = os.path.expanduser('~/.ansible-sherlock-ci.cfg')
+
+    def filter(self, targets, exclude):
+        """Filter out the cloud tests when the necessary config and resources are not available.
+        :type targets: tuple[TestTarget]
+        :type exclude: list[str]
+        """
+        if os.path.isfile(self.config_static_path):
+            return
+
+        aci = self._create_ansible_core_ci()
+
+        if os.path.isfile(aci.ci_key):
+            return
+
+        if os.path.isfile(self.SHERLOCK_CONFIG_PATH):
+            return
+
+        if is_shippable():
+            return
+
+        super(AzureCloudProvider, self).filter(targets, exclude)
+
+    def setup(self):
+        """Setup the cloud resource before delegation and register a cleanup callback."""
+        super(AzureCloudProvider, self).setup()
+
+        if not self._use_static_config():
+            self._setup_dynamic()
+
+        get_config(self.config_path)  # check required variables
+
+    def _setup_dynamic(self):
+        """Request Azure credentials through Sherlock."""
+        display.info('Provisioning %s cloud environment.' % self.platform, verbosity=1)
+
+        config = self._read_config_template()
+        response = {}
+
+        if os.path.isfile(self.SHERLOCK_CONFIG_PATH):
+            with open(self.SHERLOCK_CONFIG_PATH, 'r') as sherlock_fd:
+                sherlock_uri = sherlock_fd.readline().strip() + '&rgcount=2'
+
+            parts = urlparse(sherlock_uri)
+            query_string = parse_qs(parts.query)
+            base_uri = urlunparse(parts[:4] + ('', ''))
+
+            if 'code' not in query_string:
+                example_uri = 'https://example.azurewebsites.net/api/sandbox-provisioning'
+                raise ApplicationError('The Sherlock URI must include the API key in the query string. Example: %s?code=xxx' % example_uri)
+
+            display.info('Initializing azure/sherlock from: %s' % base_uri, verbosity=1)
+
+            http = HttpClient(self.args)
+            result = http.get(sherlock_uri)
+
+            display.info('Started azure/sherlock from: %s' % base_uri, verbosity=1)
+
+            if not self.args.explain:
+                response = result.json()
+        else:
+            aci = self._create_ansible_core_ci()
+
+            aci_result = aci.start()
+
+            if not self.args.explain:
+                response = aci_result['azure']
+
+        if not self.args.explain:
+            values = dict(
+                AZURE_CLIENT_ID=response['clientId'],
+                AZURE_SECRET=response['clientSecret'],
+                AZURE_SUBSCRIPTION_ID=response['subscriptionId'],
+                AZURE_TENANT=response['tenantId'],
+                RESOURCE_GROUP=response['resourceGroupNames'][0],
+                RESOURCE_GROUP_SECONDARY=response['resourceGroupNames'][1],
+            )
+
+            config = '\n'.join('%s: %s' % (key, values[key]) for key in sorted(values))
+
+        self._write_config(config)
+
+    def _create_ansible_core_ci(self):
+        """
+        :rtype: AnsibleCoreCI
+        """
+        return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage)
+
+
+class AzureCloudEnvironment(CloudEnvironment):
+    """Azure cloud environment plugin. Updates integration test environment after delegation."""
+    def configure_environment(self, env, cmd):
+        """
+        :type env: dict[str, str]
+        :type cmd: list[str]
+        """
+        config = get_config(self.config_path)
+
+        cmd.append('-e')
+        cmd.append('resource_prefix=%s' % self.resource_prefix)
+        cmd.append('-e')
+        cmd.append('resource_group=%s' % config['RESOURCE_GROUP'])
+        cmd.append('-e')
+        cmd.append('resource_group_secondary=%s' % config['RESOURCE_GROUP_SECONDARY'])
+
+        for key in config:
+            env[key] = config[key]
+
+    def on_failure(self, target, tries):
+        """
+        :type target: TestTarget
+        :type tries: int
+        """
+        if not tries and self.managed:
+            display.notice('If %s failed due to permissions, the test policy may need to be updated. '
+                           'For help, consult @mattclay or @gundalow on GitHub or #ansible-devel on IRC.' % target.name)
+
+    @property
+    def inventory_hosts(self):
+        """
+        :rtype: str | None
+        """
+        return 'azure'
+
+
+def get_config(config_path):
+    """
+    :param config_path: str
+    :return: dict[str, str]
+    """
+    with open(config_path, 'r') as config_fd:
+        lines = [line for line in config_fd.read().splitlines() if ':' in line and line.strip() and not line.strip().startswith('#')]
+        config = dict((kvp[0].strip(), kvp[1].strip()) for kvp in [line.split(':', 1) for line in lines])
+
+    rg_vars = (
+        'RESOURCE_GROUP',
+        'RESOURCE_GROUP_SECONDARY',
+    )
+
+    sp_vars = (
+        'AZURE_CLIENT_ID',
+        'AZURE_SECRET',
+        'AZURE_SUBSCRIPTION_ID',
+        'AZURE_TENANT',
+    )
+
+    ad_vars = (
+        'AZURE_AD_USER',
+        'AZURE_PASSWORD',
+        'AZURE_SUBSCRIPTION_ID',
+    )
+
+    rg_ok = all(var in config for var in rg_vars)
+    sp_ok = all(var in config for var in sp_vars)
+    ad_ok = all(var in config for var in ad_vars)
+
+    if not rg_ok:
+        raise ApplicationError('Resource groups must be defined with: %s' % ', '.join(sorted(rg_vars)))
+
+    if not sp_ok and not ad_ok:
+        raise ApplicationError('Credentials must be defined using either:\nService Principal: %s\nActive Directory: %s' % (
+            ', '.join(sorted(sp_vars)), ', '.join(sorted(ad_vars))))
+
+    return config

test/runner/lib/core_ci.py

@@ -56,6 +56,7 @@ class AnsibleCoreCI(object):
 
         aws_platforms = (
             'aws',
+            'azure',
             'windows',
             'freebsd',
             'rhel',

test/runner/lib/http.py

@@ -15,10 +15,10 @@ except ImportError:
 
 try:
     # noinspection PyCompatibility
-    from urlparse import urlparse
+    from urlparse import urlparse, urlunparse, parse_qs
 except ImportError:
     # noinspection PyCompatibility, PyUnresolvedReferences
-    from urllib.parse import urlparse  # pylint: disable=locally-disabled, ungrouped-imports
+    from urllib.parse import urlparse, urlunparse, parse_qs  # pylint: disable=locally-disabled, ungrouped-imports
 
 from lib.util import (
     CommonConfig,