Fix azure_rm_keyvaultkey/azure_rm_keyvaultsecret bugs (#41683)

* fix keyvault tests

* missing keyvault requirement

* fix keyvault auth bug

* apply fix in secret

* fix lint

* enable keyvault key and secret tests

* add azure service principal object_id lookup plugin

* fix lint

* add dependency in integration test

* fix bug

* put azure sp lookup plugin into test

* fix lint

* move lookup plugin

* repath lookup plugin

* repath lookup plugin

* repath files

* put az sp lookup plugin to lookup_plugins folder

lib/ansible/modules/cloud/azure/azure_rm_keyvaultkey.py

@@ -86,7 +86,7 @@ from ansible.module_utils.azure_rm_common import AzureRMModuleBase
 try:
     import re
     import codecs
-    from azure.keyvault import KeyVaultClient, KeyVaultId
+    from azure.keyvault import KeyVaultClient, KeyVaultId, KeyVaultAuthentication
     from azure.keyvault.models import KeyAttributes, JsonWebKey
     from azure.common.credentials import ServicePrincipalCredentials
     from azure.keyvault.models.key_vault_error import KeyVaultErrorException
@@ -138,7 +138,24 @@ class AzureRMKeyVaultKey(AzureRMModuleBase):
             setattr(self, key, kwargs[key])
 
         # Create KeyVaultClient
-        self.client = KeyVaultClient(self.azure_credentials)
+        def auth_callback(server, resource, scope):
+            if self.credentials['client_id'] is None or self.credentials['secret'] is None:
+                self.fail('Please specify client_id, secret and tenant to access azure Key Vault.')
+
+            tenant = self.credentials.get('tenant')
+            if not self.credentials['tenant']:
+                tenant = "common"
+
+            authcredential = ServicePrincipalCredentials(
+                client_id=self.credentials['client_id'],
+                secret=self.credentials['secret'],
+                tenant=tenant,
+                resource="https://vault.azure.net")
+
+            token = authcredential.token
+            return token['token_type'], token['access_token']
+
+        self.client = KeyVaultClient(KeyVaultAuthentication(auth_callback))
 
         results = dict()
         changed = False
@@ -187,7 +204,7 @@ class AzureRMKeyVaultKey(AzureRMModuleBase):
 
     def create_key(self, name, tags, kty='RSA'):
         ''' Creates a key '''
-        key_bundle = self.client.create_key(self.keyvault_uri, name, kty, tags=tags)
+        key_bundle = self.client.create_key(vault_base_url=self.keyvault_uri, key_name=name, kty=kty, tags=tags)
         key_id = KeyVaultId.parse_key_id(key_bundle.key.kid)
         return key_id.id
 

lib/ansible/modules/cloud/azure/azure_rm_keyvaultsecret.py

@@ -133,7 +133,24 @@ class AzureRMKeyVaultSecret(AzureRMModuleBase):
             setattr(self, key, kwargs[key])
 
         # Create KeyVault Client using KeyVault auth class and auth_callback
-        self.client = KeyVaultClient(self.azure_credentials)
+        def auth_callback(server, resource, scope):
+            if self.credentials['client_id'] is None or self.credentials['secret'] is None:
+                self.fail('Please specify client_id, secret and tenant to access azure Key Vault.')
+
+            tenant = self.credentials.get('tenant')
+            if not self.credentials['tenant']:
+                tenant = "common"
+
+            authcredential = ServicePrincipalCredentials(
+                client_id=self.credentials['client_id'],
+                secret=self.credentials['secret'],
+                tenant=tenant,
+                resource="https://vault.azure.net")
+
+            token = authcredential.token
+            return token['token_type'], token['access_token']
+
+        self.client = KeyVaultClient(KeyVaultAuthentication(auth_callback))
 
         results = dict()
         changed = False

packaging/requirements/requirements-azure.txt

@@ -21,3 +21,5 @@ azure-nspkg==2.0.0
 azure-storage==0.35.1
 msrest==0.4.29
 msrestazure==0.4.31
+azure-keyvault==1.0.0a1
+azure-graphrbac==0.40.0

test/integration/targets/azure_rm_keyvaultkey/aliases

@@ -1,4 +1,3 @@
 cloud/azure
 posix/ci/cloud/group2/azure
 destructive
-disabled

test/integration/targets/azure_rm_keyvaultkey/lookup_plugins/azure_service_principal_attribute.py

@@ -0,0 +1,94 @@
+# (c) 2018 Yunge Zhu, <yungez@microsoft.com>
+# (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = """
+lookup: azure_service_principal_attribute
+
+requirements:
+  - azure-graphrbac
+
+author:
+  - Yunge Zhu <yungez@microsoft.com>
+
+version_added: "2.7"
+
+short_description: Look up Azure service principal attributes.
+
+description:
+  - Describes object id of your Azure service principal account.
+options:
+  azure_client_id:
+    description: azure service principal client id.
+  azure_secret:
+    description: azure service principal secret
+  azure_tenant:
+    description: azure tenant
+  azure_cloud_environment:
+    description: azure cloud environment
+"""
+
+EXAMPLES = """
+set_fact:
+  object_id: "{{ lookup('azure_service_principal_attribute',
+                         azure_client_id=azure_client_id,
+                         azure_secret=azure_secret,
+                         azure_tenant=azure_secret) }}"
+"""
+
+RETURN = """
+_raw:
+  description:
+    Returns object id of service principal.
+"""
+
+from ansible.errors import AnsibleError
+from ansible.plugins import AnsiblePlugin
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils._text import to_native
+
+try:
+    from azure.common.credentials import ServicePrincipalCredentials
+    from azure.graphrbac import GraphRbacManagementClient
+    from msrestazure import azure_cloud
+    from msrestazure.azure_exceptions import CloudError
+except ImportError:
+    raise AnsibleError(
+        "The lookup azure_service_principal_attribute requires azure.graphrbac, msrest")
+
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables, **kwargs):
+
+        self.set_options(direct=kwargs)
+
+        credentials = {}
+        credentials['azure_client_id'] = self.get_option('azure_client_id', None)
+        credentials['azure_secret'] = self.get_option('azure_secret', None)
+        credentials['azure_tenant'] = self.get_option('azure_tenant', 'common')
+
+        if credentials['azure_client_id'] is None or credentials['azure_secret'] is None:
+            raise AnsibleError("Must specify azure_client_id and azure_secret")
+
+        _cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD
+        if self.get_option('azure_cloud_environment', None) is not None:
+            cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(credentials['azure_cloud_environment'])
+
+        try:
+            azure_credentials = ServicePrincipalCredentials(client_id=credentials['azure_client_id'],
+                                                            secret=credentials['azure_secret'],
+                                                            tenant=credentials['azure_tenant'],
+                                                            resource=_cloud_environment.endpoints.active_directory_graph_resource_id)
+
+            client = GraphRbacManagementClient(azure_credentials, credentials['azure_tenant'],
+                                               base_url=_cloud_environment.endpoints.active_directory_graph_resource_id)
+
+            response = list(client.service_principals.list(filter="appId eq '{0}'".format(credentials['azure_client_id'])))
+            sp = response[0]
+
+            return sp.object_id.split(',')
+        except CloudError as ex:
+            raise AnsibleError("Failed to get service principal object id: %s" % to_native(ex))
+        return False

test/integration/targets/azure_rm_keyvaultkey/tasks/main.yml

@@ -1,20 +1,35 @@
 - name: Prepare random number
   set_fact:
     rpfx: "{{ resource_group | hash('md5') | truncate(7, True, '') }}{{ 1000 | random }}"
+    tenant_id: "{{ lookup('env','AZURE_TENANT') }}"
   run_once: yes
 
+- name: set service principal info
+  set_fact:
+    azure_client_id: "{{ lookup('env','AZURE_CLIENT_ID') }}"
+    azure_secret: "{{ lookup('env','AZURE_SECRET') }}"
+  no_log: yes
+
+- name: lookup service principal object id
+  set_fact:
+    object_id: "{{ lookup('azure_service_principal_attribute',
+                   azure_client_id=azure_client_id,
+                   azure_secret=azure_secret,
+                   azure_tenant=tenant_id) }}"
+  register: object_id
+
 - name: Create instance of Key Vault
   azure_rm_keyvault:
     resource_group: "{{ resource_group }}"
     vault_name: "vault{{ rpfx }}"
     enabled_for_deployment: yes
-    vault_tenant: "{{ azure_tenant }}"
+    vault_tenant: "{{ tenant_id }}"
     sku:
       name: standard
       family: A
     access_policies:
-      - tenant_id: "{{ azure_tenant }}"
-        object_id: 97567bfa-cf13-4217-8fa3-cc56bc1867fe
+      - tenant_id: "{{ tenant_id }}"
+        object_id: '{{ object_id }}'
         keys:
           - get
           - list
@@ -25,6 +40,12 @@
           - recover
           - backup
           - restore
+          - encrypt
+          - decrypt
+          - wrapkey
+          - unwrapkey
+          - sign
+          - verify
         secrets:
           - get
           - list

test/integration/targets/azure_rm_keyvaultsecret/aliases

@@ -1,4 +1,3 @@
 cloud/azure
 posix/ci/cloud/group2/azure
-destructive
-disabled
+destructive

test/integration/targets/azure_rm_keyvaultsecret/lookup_plugins/azure_service_principal_attribute.py

@@ -0,0 +1,94 @@
+# (c) 2018 Yunge Zhu, <yungez@microsoft.com>
+# (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = """
+lookup: azure_service_principal_attribute
+
+requirements:
+  - azure-graphrbac
+
+author:
+  - Yunge Zhu <yungez@microsoft.com>
+
+version_added: "2.7"
+
+short_description: Look up Azure service principal attributes.
+
+description:
+  - Describes object id of your Azure service principal account.
+options:
+  azure_client_id:
+    description: azure service principal client id.
+  azure_secret:
+    description: azure service principal secret
+  azure_tenant:
+    description: azure tenant
+  azure_cloud_environment:
+    description: azure cloud environment
+"""
+
+EXAMPLES = """
+set_fact:
+  object_id: "{{ lookup('azure_service_principal_attribute',
+                         azure_client_id=azure_client_id,
+                         azure_secret=azure_secret,
+                         azure_tenant=azure_secret) }}"
+"""
+
+RETURN = """
+_raw:
+  description:
+    Returns object id of service principal.
+"""
+
+from ansible.errors import AnsibleError
+from ansible.plugins import AnsiblePlugin
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils._text import to_native
+
+try:
+    from azure.common.credentials import ServicePrincipalCredentials
+    from azure.graphrbac import GraphRbacManagementClient
+    from msrestazure import azure_cloud
+    from msrestazure.azure_exceptions import CloudError
+except ImportError:
+    raise AnsibleError(
+        "The lookup azure_service_principal_attribute requires azure.graphrbac, msrest")
+
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables, **kwargs):
+
+        self.set_options(direct=kwargs)
+
+        credentials = {}
+        credentials['azure_client_id'] = self.get_option('azure_client_id', None)
+        credentials['azure_secret'] = self.get_option('azure_secret', None)
+        credentials['azure_tenant'] = self.get_option('azure_tenant', 'common')
+
+        if credentials['azure_client_id'] is None or credentials['azure_secret'] is None:
+            raise AnsibleError("Must specify azure_client_id and azure_secret")
+
+        _cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD
+        if self.get_option('azure_cloud_environment', None) is not None:
+            cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(credentials['azure_cloud_environment'])
+
+        try:
+            azure_credentials = ServicePrincipalCredentials(client_id=credentials['azure_client_id'],
+                                                            secret=credentials['azure_secret'],
+                                                            tenant=credentials['azure_tenant'],
+                                                            resource=_cloud_environment.endpoints.active_directory_graph_resource_id)
+
+            client = GraphRbacManagementClient(azure_credentials, credentials['azure_tenant'],
+                                               base_url=_cloud_environment.endpoints.active_directory_graph_resource_id)
+
+            response = list(client.service_principals.list(filter="appId eq '{0}'".format(credentials['azure_client_id'])))
+            sp = response[0]
+
+            return sp.object_id.split(',')
+        except CloudError as ex:
+            raise AnsibleError("Failed to get service principal object id: %s" % to_native(ex))
+        return False

test/integration/targets/azure_rm_keyvaultsecret/tasks/main.yml

@@ -1,20 +1,35 @@
 - name: Prepare random number
   set_fact:
     rpfx: "{{ resource_group | hash('md5') | truncate(7, True, '') }}{{ 1000 | random }}"
+    tenant_id: "{{ lookup('env','AZURE_TENANT') }}"
   run_once: yes
 
+- name: set service principal info
+  set_fact:
+    azure_client_id: "{{ lookup('env','AZURE_CLIENT_ID') }}"
+    azure_secret: "{{ lookup('env','AZURE_SECRET') }}"
+  no_log: yes
+
+- name: lookup service principal object id
+  set_fact:
+    object_id: "{{ lookup('azure_service_principal_attribute',
+                   azure_client_id=azure_client_id,
+                   azure_secret=azure_secret,
+                   azure_tenant=tenant_id) }}"
+  register: object_id
+
 - name: Create instance of Key Vault
   azure_rm_keyvault:
     resource_group: "{{ resource_group }}"
     vault_name: "vault{{ rpfx }}"
     enabled_for_deployment: yes
-    vault_tenant: "{{ azure_tenant }}"
+    vault_tenant: "{{ tenant_id }}"
     sku:
       name: standard
       family: A
     access_policies:
-      - tenant_id: "{{ azure_tenant }}"
-        object_id: 97567bfa-cf13-4217-8fa3-cc56bc1867fe
+      - tenant_id: "{{ tenant_id }}"
+        object_id: "{{ object_id }}"
         keys:
           - get
           - list

test/runner/requirements/integration.cloud.azure.txt

@@ -21,3 +21,5 @@ azure-nspkg==2.0.0
 azure-storage==0.35.1
 msrest==0.4.29
 msrestazure==0.4.31
+azure-keyvault==1.0.0a1
+azure-graphrbac==0.40.0