expose cloud_environment override in azure_rm modules (#28743)

* Can be set via env, credential profile, or module arg
* Valid values defined by Azure Python SDK, currently `AzureCloud`,`AzureChinaCloud`,`AzureUSGovernment`,`AzureGermanCloud` or any Azure Stack metadata discovery URL.

CHANGELOG.md

@@ -172,6 +172,8 @@ Ansible Changes By Release
   template hardcoded this to true.
 - Added a new parameter to command module that lets users specify data to pipe
   into the command's stdin.
+- The azure_rm modules now accept a `cloud_environment` arg to access regional and private clouds.
+- The azure_rm modules now require at least version 2.0.0 of the Azure Python SDK.
 
 ### New Modules
 

contrib/inventory/azure_rm.py

@@ -49,6 +49,7 @@ Command line arguments:
  - tenant
  - ad_user
  - password
+ - cloud_environment
 
 Environment variables:
  - AZURE_PROFILE
@@ -58,6 +59,7 @@ Environment variables:
  - AZURE_TENANT
  - AZURE_AD_USER
  - AZURE_PASSWORD
+ - AZURE_CLOUD_ENVIRONMENT
 
 Run for Specific Host
 -----------------------
@@ -190,22 +192,27 @@ import json
 import os
 import re
 import sys
+import inspect
+import traceback
+
 
 from packaging.version import Version
 
 from os.path import expanduser
+import ansible.module_utils.six.moves.urllib.parse as urlparse
 
 HAS_AZURE = True
 HAS_AZURE_EXC = None
 
 try:
     from msrestazure.azure_exceptions import CloudError
+    from msrestazure import azure_cloud
     from azure.mgmt.compute import __version__ as azure_compute_version
     from azure.common import AzureMissingResourceHttpError, AzureHttpError
     from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
-    from azure.mgmt.network.network_management_client import NetworkManagementClient
+    from azure.mgmt.network import NetworkManagementClient
-    from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient
+    from azure.mgmt.resource.resources import ResourceManagementClient
-    from azure.mgmt.compute.compute_management_client import ComputeManagementClient
+    from azure.mgmt.compute import ComputeManagementClient
 except ImportError as exc:
     HAS_AZURE_EXC = exc
     HAS_AZURE = False
@@ -218,7 +225,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict(
     secret='AZURE_SECRET',
     tenant='AZURE_TENANT',
     ad_user='AZURE_AD_USER',
-    password='AZURE_PASSWORD'
+    password='AZURE_PASSWORD',
+    cloud_environment='AZURE_CLOUD_ENVIRONMENT',
 )
 
 AZURE_CONFIG_SETTINGS = dict(
@@ -232,7 +240,7 @@ AZURE_CONFIG_SETTINGS = dict(
     group_by_tag='AZURE_GROUP_BY_TAG'
 )
 
-AZURE_MIN_VERSION = "0.30.0rc5"
+AZURE_MIN_VERSION = "2.0.0"
 
 
 def azure_id_to_dict(id):
@@ -249,6 +257,7 @@ class AzureRM(object):
 
     def __init__(self, args):
         self._args = args
+        self._cloud_environment = None
         self._compute_client = None
         self._resource_client = None
         self._network_client = None
@@ -262,6 +271,26 @@ class AzureRM(object):
             self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
                       "or define a profile in ~/.azure/credentials.")
 
+        # if cloud_environment specified, look up/build Cloud object
+        raw_cloud_env = self.credentials.get('cloud_environment')
+        if not raw_cloud_env:
+            self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD  # SDK default
+        else:
+            # try to look up "well-known" values via the name attribute on azure_cloud members
+            all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
+            matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
+            if len(matched_clouds) == 1:
+                self._cloud_environment = matched_clouds[0]
+            elif len(matched_clouds) > 1:
+                self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
+            else:
+                if not urlparse.urlparse(raw_cloud_env).scheme:
+                    self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
+                try:
+                    self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
+                except Exception as e:
+                    self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message))
+
         if self.credentials.get('subscription_id', None) is None:
             self.fail("Credentials did not include a subscription_id value.")
         self.log("setting subscription_id")
@@ -272,13 +301,16 @@ class AzureRM(object):
            self.credentials.get('tenant') is not None:
             self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
                                                                  secret=self.credentials['secret'],
-                                                                 tenant=self.credentials['tenant'])
+                                                                 tenant=self.credentials['tenant'],
+                                                                 cloud_environment=self._cloud_environment)
         elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
             tenant = self.credentials.get('tenant')
-            if tenant is not None:
+            if not tenant:
-                self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'], tenant=tenant)
+                tenant = 'common'
-            else:
+            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
-                self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
+                                                         self.credentials['password'],
+                                                         tenant=tenant,
+                                                         cloud_environment=self._cloud_environment)
         else:
             self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
                       "Credentials must include client_id, secret and tenant or ad_user and password.")
@@ -345,6 +377,10 @@ class AzureRM(object):
             self.log('Received credentials from parameters.')
             return arg_credentials
 
+        if arg_credentials['ad_user'] is not None:
+            self.log('Received credentials from parameters.')
+            return arg_credentials
+
         # try environment
         env_credentials = self._get_env_credentials()
         if env_credentials:
@@ -376,7 +412,7 @@ class AzureRM(object):
     def network_client(self):
         self.log('Getting network client')
         if not self._network_client:
-            self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id)
+            self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id, base_url=self._cloud_environment.endpoints.management)
             self._register('Microsoft.Network')
         return self._network_client
 
@@ -384,14 +420,16 @@ class AzureRM(object):
     def rm_client(self):
         self.log('Getting resource manager client')
         if not self._resource_client:
-            self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id)
+            self._resource_client = ResourceManagementClient(self.azure_credentials,
+                                                             self.subscription_id,
+                                                             base_url=self._cloud_environment.endpoints.management)
         return self._resource_client
 
     @property
     def compute_client(self):
         self.log('Getting compute client')
         if not self._compute_client:
-            self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id)
+            self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id, base_url=self._cloud_environment.endpoints.management)
             self._register('Microsoft.Compute')
         return self._compute_client
 
@@ -469,10 +507,12 @@ class AzureInventory(object):
                             help='Azure Client Secret')
         parser.add_argument('--tenant', action='store',
                             help='Azure Tenant Id')
-        parser.add_argument('--ad-user', action='store',
+        parser.add_argument('--ad_user', action='store',
                             help='Active Directory User')
         parser.add_argument('--password', action='store',
                             help='password')
+        parser.add_argument('--cloud_environment', action='store',
+                            help='Azure Cloud Environment name or metadata discovery URL')
         parser.add_argument('--resource-groups', action='store',
                             help='Return inventory for comma separated list of resource group names')
         parser.add_argument('--tags', action='store',
@@ -793,11 +833,7 @@ class AzureInventory(object):
 
 def main():
     if not HAS_AZURE:
-        sys.exit("The Azure python sdk is not installed (try `pip install 'azure>=2.0.0rc5' --upgrade`) - {0}".format(HAS_AZURE_EXC))
+        sys.exit("The Azure python sdk is not installed (try `pip install 'azure>={0}' --upgrade`) - {1}".format(AZURE_MIN_VERSION, HAS_AZURE_EXC))
-
-    if Version(azure_compute_version) < Version(AZURE_MIN_VERSION):
-        sys.exit("Expecting azure.mgmt.compute.__version__ to be {0}. Found version {1} "
-                 "Do you have Azure >= 2.0.0rc5 installed? (try `pip install 'azure>=2.0.0rc5' --upgrade`)".format(AZURE_MIN_VERSION, azure_compute_version))
 
     AzureInventory()
 

docs/docsite/rst/guide_azure.rst

@@ -119,6 +119,14 @@ Or, pass the following parameters for Active Directory username/password:
 * subscription_id
 
 
+Other Cloud Environments
+------------------------
+
+To use an Azure Cloud other than the default public cloud (eg, Azure China Cloud, Azure US Government Cloud, Azure Stack),
+pass the "cloud_environment" argument to modules, configure it in a credential profile, or set the "AZURE_CLOUD_ENVIRONMENT"
+environment variable. The value is either a cloud name as defined by the Azure Python SDK (eg, "AzureChinaCloud",
+"AzureUSGovernment"; defaults to "AzureCloud") or an Azure metadata discovery URL (for Azure Stack).
+
 Creating Virtual Machines
 -------------------------
 

docs/docsite/rst/roadmap/ROADMAP_2_4.rst

@@ -174,8 +174,8 @@ AWS
 
 Azure
 -----
-- Expose endpoint overrides **(in progress)**
+- Expose endpoint overrides **(done)**
-- Reformat/document module output to collapse internal API structures and surface important data (eg, public IPs, NICs, data disks)
+- Reformat/document module output to collapse internal API structures and surface important data (eg, public IPs, NICs, data disks) **(pushed to future)**
 - Add load balancer module **(in progress)**
 - Add Azure Functions module **(in progress)**
 

lib/ansible/module_utils/azure_rm_common.py

@@ -23,12 +23,14 @@ import sys
 import copy
 import importlib
 import inspect
+import traceback
 
 from packaging.version import Version
 from os.path import expanduser
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six.moves import configparser
+import ansible.module_utils.six.moves.urllib.parse as urlparse
 
 AZURE_COMMON_ARGS = dict(
     cli_default_profile=dict(type='bool'),
@@ -39,6 +41,7 @@ AZURE_COMMON_ARGS = dict(
     tenant=dict(type='str', no_log=True),
     ad_user=dict(type='str', no_log=True),
     password=dict(type='str', no_log=True),
+    cloud_environment=dict(type='str'),
     # debug=dict(type='bool', default=False),
 )
 
@@ -50,7 +53,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict(
     secret='AZURE_SECRET',
     tenant='AZURE_TENANT',
     ad_user='AZURE_AD_USER',
-    password='AZURE_PASSWORD'
+    password='AZURE_PASSWORD',
+    cloud_environment='AZURE_CLOUD_ENVIRONMENT',
 )
 
 AZURE_TAG_ARGS = dict(
@@ -87,6 +91,7 @@ except ImportError as exc:
 try:
     from enum import Enum
     from msrestazure.azure_exceptions import CloudError
+    from msrestazure import azure_cloud
     from azure.mgmt.network.models import PublicIPAddress, NetworkSecurityGroup, SecurityRule, NetworkInterface, \
         NetworkInterfaceIPConfiguration, Subnet
     from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
@@ -173,6 +178,7 @@ class AzureRMModuleBase(object):
             self.fail("Do you have azure>={1} installed? Try `pip install 'azure>={1}' --upgrade`"
                       "- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))
 
+        self._cloud_environment = None
         self._network_client = None
         self._storage_client = None
         self._resource_client = None
@@ -188,6 +194,26 @@ class AzureRMModuleBase(object):
             self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
                       "or define a profile in ~/.azure/credentials or be logged using AzureCLI.")
 
+        # if cloud_environment specified, look up/build Cloud object
+        raw_cloud_env = self.credentials.get('cloud_environment')
+        if not raw_cloud_env:
+            self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD  # SDK default
+        else:
+            # try to look up "well-known" values via the name attribute on azure_cloud members
+            all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
+            matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
+            if len(matched_clouds) == 1:
+                self._cloud_environment = matched_clouds[0]
+            elif len(matched_clouds) > 1:
+                self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
+            else:
+                if not urlparse.urlparse(raw_cloud_env).scheme:
+                    self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
+                try:
+                    self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
+                except Exception as e:
+                    self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
+
         if self.credentials.get('subscription_id', None) is None:
             self.fail("Credentials did not include a subscription_id value.")
         self.log("setting subscription_id")
@@ -198,24 +224,23 @@ class AzureRMModuleBase(object):
            self.credentials.get('tenant') is not None:
             self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
                                                                  secret=self.credentials['secret'],
-                                                                 tenant=self.credentials['tenant'])
+                                                                 tenant=self.credentials['tenant'],
+                                                                 cloud_environment=self._cloud_environment)
+
         elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
             tenant = self.credentials.get('tenant')
-            if tenant is not None:
+            if not tenant:
-                self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'], tenant=tenant)
+                tenant = 'common'  # SDK default
-            else:
+
-                self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
+            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
+                                                         self.credentials['password'],
+                                                         tenant=tenant,
+                                                         cloud_environment=self._cloud_environment)
         else:
             self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
                       "Credentials must include client_id, secret and tenant or ad_user and password or "
                       "be logged using AzureCLI.")
 
-        # base_url for sovereign cloud support. For now only if AzureCLI
-        if self.credentials.get('base_url') is not None:
-            self.base_url = self.credentials.get('base_url')
-        else:
-            self.base_url = None
-
         # common parameter validation
         if self.module.params.get('tags'):
             self.validate_tags(self.module.params['tags'])
@@ -340,11 +365,10 @@ class AzureRMModuleBase(object):
             self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`")
         try:
             credentials, subscription_id = get_azure_cli_credentials()
-            base_url = get_cli_active_cloud().endpoints.resource_manager
+            self._cloud_environment = get_cli_active_cloud()
             return {
                 'credentials': credentials,
-                'subscription_id': subscription_id,
+                'subscription_id': subscription_id
-                'base_url': base_url
             }
         except CLIError as err:
             self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
@@ -641,7 +665,7 @@ class AzureRMModuleBase(object):
             self._storage_client = StorageManagementClient(
                 self.azure_credentials,
                 self.subscription_id,
-                base_url=self.base_url,
+                base_url=self._cloud_environment.endpoints.resource_manager,
                 api_version='2017-06-01'
             )
             self._register('Microsoft.Storage')
@@ -655,7 +679,7 @@ class AzureRMModuleBase(object):
             self._network_client = NetworkManagementClient(
                 self.azure_credentials,
                 self.subscription_id,
-                base_url=self.base_url,
+                base_url=self._cloud_environment.endpoints.resource_manager,
                 api_version='2017-06-01'
             )
             self._register('Microsoft.Network')
@@ -669,7 +693,7 @@ class AzureRMModuleBase(object):
             self._resource_client = ResourceManagementClient(
                 self.azure_credentials,
                 self.subscription_id,
-                base_url=self.base_url,
+                base_url=self._cloud_environment.endpoints.resource_manager,
                 api_version='2017-05-10'
             )
         return self._resource_client
@@ -682,7 +706,7 @@ class AzureRMModuleBase(object):
             self._compute_client = ComputeManagementClient(
                 self.azure_credentials,
                 self.subscription_id,
-                base_url=self.base_url,
+                base_url=self._cloud_environment.endpoints.resource_manager,
                 api_version='2017-03-30'
             )
             self._register('Microsoft.Compute')
@@ -696,7 +720,7 @@ class AzureRMModuleBase(object):
             self._dns_client = DnsManagementClient(
                 self.azure_credentials,
                 self.subscription_id,
-                base_url=self.base_url
+                base_url=self._cloud_environment.endpoints.resource_manager,
             )
             self._register('Microsoft.Dns')
         return self._dns_client

lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py

@@ -430,10 +430,10 @@ AZURE_OBJECT_CLASS = 'VirtualMachine'
 AZURE_ENUM_MODULES = ['azure.mgmt.compute.models']
 
 
-def extract_names_from_blob_uri(blob_uri):
+def extract_names_from_blob_uri(blob_uri, storage_suffix):
     # HACK: ditch this once python SDK supports get by URI
-    m = re.match('^https://(?P<accountname>[^\.]+)\.blob\.core\.windows\.net/'
+    m = re.match('^https://(?P<accountname>[^\.]+)\.blob\.{0}/'
-                 '(?P<containername>[^/]+)/(?P<blobname>.+)$', blob_uri)
+                 '(?P<containername>[^/]+)/(?P<blobname>.+)$'.format(storage_suffix), blob_uri)
     if not m:
         raise Exception("unable to parse blob uri '%s'" % blob_uri)
     extracted_names = m.groupdict()
@@ -574,7 +574,8 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
             if self.storage_account_name:
                 self.get_storage_account(self.storage_account_name)
 
-                requested_vhd_uri = 'https://{0}.blob.core.windows.net/{1}/{2}'.format(self.storage_account_name,
+                requested_vhd_uri = 'https://{0}.blob.{1}/{2}/{3}'.format(self.storage_account_name,
+                                                                          self._cloud_environment.suffixes.storage_endpoint,
                                                                           self.storage_container_name,
                                                                           self.storage_blob_name)
 
@@ -689,8 +690,9 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
                         storage_account = self.create_default_storage_account()
                         self.log("storage account:")
                         self.log(self.serialize_obj(storage_account, 'StorageAccount'), pretty_print=True)
-                        requested_vhd_uri = 'https://{0}.blob.core.windows.net/{1}/{2}'.format(
+                        requested_vhd_uri = 'https://{0}.blob.{1}/{2}/{3}'.format(
                             storage_account.name,
+                            self._cloud_environment.suffixes.storage_endpoint,
                             self.storage_container_name,
                             self.storage_blob_name)
 
@@ -767,7 +769,7 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
                                 vm_dict['properties']['storageProfile']['osDisk']['name'],
                                 vhd,
                                 vm_dict['properties']['storageProfile']['osDisk']['createOption'],
-                                os_type=vm_dict['properties']['storageProfile']['osDisk']['osType'],
+                                vm_dict['properties']['storageProfile']['osDisk']['osType'],
                                 caching=vm_dict['properties']['storageProfile']['osDisk']['caching']
                             ),
                             image_reference=ImageReference(
@@ -1028,7 +1030,7 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
         for uri in vhd_uris:
             self.log("Extracting info from blob uri '{0}'".format(uri))
             try:
-                blob_parts = extract_names_from_blob_uri(uri)
+                blob_parts = extract_names_from_blob_uri(uri, self._cloud_environment.suffixes.storage_endpoint)
             except Exception as exc:
                 self.fail("Error parsing blob URI {0}".format(str(exc)))
             storage_account_name = blob_parts['accountname']

lib/ansible/utils/module_docs_fragments/azure.py

@@ -61,17 +61,22 @@ options:
             - Azure tenant ID. Use when authenticating with a Service Principal.
         required: false
         default: null
-
+    cloud_environment:
+        description:
+            - For cloud environments other than the US public cloud, the environment name (as defined by Azure Python SDK, eg, C(AzureChinaCloud),
+              C(AzureUSGovernment)), or a metadata discovery endpoint URL (required for Azure Stack). Can also be set via credential file profile or
+              the C(AZURE_CLOUD_ENVIRONMENT) environment variable.
+        default: AzureCloud
 requirements:
     - "python >= 2.7"
-    - "azure == 2.0.0rc5"
+    - "azure >= 2.0.0"
 
 notes:
     - For authentication with Azure you can pass parameters, set environment variables or use a profile stored
       in ~/.azure/credentials. Authentication is possible using a service principal or Active Directory user.
-      To authenticate via service principal pass subscription_id, client_id, secret and tenant or set set environment
+      To authenticate via service principal, pass subscription_id, client_id, secret and tenant or set environment
       variables AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID, AZURE_SECRET and AZURE_TENANT.
-    - To Authentication via Active Directory user pass ad_user and password, or set AZURE_AD_USER and
+    - To authenticate via Active Directory user, pass ad_user and password, or set AZURE_AD_USER and
       AZURE_PASSWORD in the environment.
     - "Alternatively, credentials can be stored in ~/.azure/credentials. This is an ini file containing
       a [default] section and the following keys: subscription_id, client_id, secret and tenant or