Add support for HashiCorp Vault JWT auth (#1213)

* Add support for Hashicorp Vault JWT auth

* Add support for HashiCorp Vault JWT auth (continued)

Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com>

Co-authored-by: Mike Brancato <mike@mikebrancato.com>
Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com>

changelogs/fragments/1213-hashi_vault-jwt-auth-support.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+  - hashi_vault lookup plugin - add support for JWT authentication (https://github.com/ansible-collections/community.general/pull/1213).

plugins/lookup/hashi_vault.py

@@ -11,7 +11,7 @@ DOCUMENTATION = """
   author:
     - Jonathan Davila (!UNKNOWN) <jdavila(at)ansible.com>
     - Brian Scholer (@briantist)
-  short_description: Retrieve secrets from HashiCorp's vault
+  short_description: Retrieve secrets from HashiCorp's Vault
   requirements:
     - hvac (python library)
     - hvac 0.7.0+ (for namespace support)
@@ -19,7 +19,7 @@ DOCUMENTATION = """
     - botocore (only if inferring aws params from boto)
     - boto3 (only if using a boto profile)
   description:
-    - Retrieve secrets from HashiCorp's vault.
+    - Retrieve secrets from HashiCorp's Vault.
   notes:
     - Due to a current limitation in the HVAC library there won't necessarily be an error if a bad endpoint is specified.
     - As of community.general 0.2.0, only the latest version of a secret is returned when specifying a KV v2 path.
@@ -27,7 +27,7 @@ DOCUMENTATION = """
     - As of community.general 0.2.0, when C(secret) is the first option in the term string, C(secret=) is not required (see examples).
   options:
     secret:
-      description: query you are making.
+      description: Vault path to the secret being requested in the format C(path[:field]).
       required: True
     token:
       description:
@@ -55,7 +55,7 @@ DOCUMENTATION = """
       default: '.vault-token'
       version_added: '0.2.0'
     url:
-      description: URL to vault service.
+      description: URL to the Vault service.
       env:
         - name: VAULT_ADDR
       ini:
@@ -76,7 +76,7 @@ DOCUMENTATION = """
           key: role_id
           version_added: '0.2.0'
     secret_id:
-      description: Secret id for a vault AppRole auth.
+      description: Secret ID to be used for Vault AppRole authentication.
       env:
         - name: VAULT_SECRET_ID
     auth_method:
@@ -84,6 +84,7 @@ DOCUMENTATION = """
         - Authentication method to be used.
         - C(userpass) is added in Ansible 2.8.
         - C(aws_iam_login) is added in community.general 0.2.0.
+        - C(jwt) is added in community.general 1.3.0.
       env:
         - name: VAULT_AUTH_METHOD
       ini:
@@ -96,6 +97,7 @@ DOCUMENTATION = """
         - ldap
         - approle
         - aws_iam_login
+        - jwt
       default: token
     return_format:
       description:
@@ -111,7 +113,12 @@ DOCUMENTATION = """
       aliases: [ as ]
       version_added: '0.2.0'
     mount_point:
-      description: Vault mount point, only required if you have a custom mount point.
+      description: Vault mount point, only required if you have a custom mount point. Does not apply to token authentication.
+    jwt:
+      description: The JSON Web Token (JWT) to use for JWT authentication to Vault.
+      env:
+        - name: ANSIBLE_HASHI_VAULT_JWT
+      version_added: 1.3.0
     ca_cert:
       description: Path to certificate to use for authentication.
       aliases: [ cacert ]
@@ -123,7 +130,10 @@ DOCUMENTATION = """
         - Will default to C(true) if neither I(validate_certs) or C(VAULT_SKIP_VERIFY) are set.
       type: boolean
     namespace:
-      description: Namespace where secrets reside. Requires HVAC 0.7.0+ and Vault 0.11+.
+      description:
+        - Vault namespace where secrets reside. This option requires HVAC 0.7.0+ and Vault 0.11+.
+        - Optionally, this may be achieved by prefixing the authentication mount point and/or secret path with the namespace
+          (e.g C(mynamespace/secret/mysecret)).
       env:
         - name: VAULT_NAMESPACE
           version_added: 1.2.0
@@ -186,7 +196,7 @@ EXAMPLES = """
   ansible.builtin.debug:
     msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello:value auth_method=userpass username=myuser password=psw url=http://myvault:8200') }}"
 
-- name: Using an ssl vault
+- name: Connect to Vault using TLS
   ansible.builtin.debug:
     msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hola:value token=c975b780-d1be-8016-866b-01d0f9b688a5 validate_certs=False') }}"
 
@@ -194,7 +204,7 @@ EXAMPLES = """
   ansible.builtin.debug:
     msg: "{{ lookup('community.general.hashi_vault', 'secret/hi:value token=xxxx url=https://myvault:8200 validate_certs=True cacert=/cacert/path/ca.pem') }}"
 
-- name: authenticate with a Vault app role
+- name: Authenticate with a Vault app role
   ansible.builtin.debug:
     msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello:value auth_method=approle role_id=myroleid secret_id=mysecretid') }}"
 
@@ -244,7 +254,13 @@ EXAMPLES = """
 
 - name: authenticate with aws_iam_login
   ansible.builtin.debug:
-    msg: "{{ lookup('community.general.hashi_vault', 'secret/hello:value', auth_method='aws_iam_login' role_id='myroleid', profile=my_boto_profile) }}"
+    msg: "{{ lookup('community.general.hashi_vault', 'secret/hello:value', auth_method='aws_iam_login', role_id='myroleid', profile=my_boto_profile) }}"
+
+# The following examples work in collection releases after community.general 1.3.0
+
+- name: Authenticate with a JWT
+  ansible.builtin.debug:
+      msg: "{{ lookup('community.general.hashi_vault', 'secret/hello:value', auth_method='jwt', role_id='myroleid', jwt='myjwt', url='https://myvault:8200')}}"
 """
 
 RETURN = """
@@ -428,6 +444,17 @@ class HashiVault:
             Display().warning("HVAC should be updated to version 0.9.3 or higher. Deprecated method 'auth_aws_iam' will be used.")
             self.client.auth_aws_iam(**params)
 
+    def auth_jwt(self):
+        params = self.get_options('role_id', 'jwt', 'mount_point')
+        params['role'] = params.pop('role_id')
+        if self.hvac_has_auth_methods and hasattr(self.client.auth, 'jwt') and hasattr(self.client.auth.jwt, 'jwt_login'):
+            response = self.client.auth.jwt.jwt_login(**params)
+            # must manually set the client token with JWT login
+            # see https://github.com/hvac/hvac/issues/644
+            self.client.token = response['auth']['client_token']
+        else:
+            raise AnsibleError("JWT authentication requires HVAC version 0.10.5 or higher.")
+
     # end auth implementation methods
 
 
@@ -530,7 +557,7 @@ class LookupModule(LookupBase):
     def auth_methods(self):
         # enforce and set the list of available auth methods
         # TODO: can this be read from the choices: field in documentation?
-        avail_auth_methods = ['token', 'approle', 'userpass', 'ldap', 'aws_iam_login']
+        avail_auth_methods = ['token', 'approle', 'userpass', 'ldap', 'aws_iam_login', 'jwt']
         self.set_option('avail_auth_methods', avail_auth_methods)
         auth_method = self.get_option('auth_method')
 
@@ -617,4 +644,7 @@ class LookupModule(LookupBase):
 
         self.set_option('iam_login_credentials', params)
 
+    def validate_auth_jwt(self, auth_method):
+        self.validate_by_required_fields(auth_method, 'role_id', 'jwt')
+
     # end auth method validators

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/jwt_private.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAnzyis1ZjfNB0bBgKFMSvvkTtwlvBsaJq7S5wA+kzeVOVpVWw
+kWdVha4s38XM/pa/yr47av7+z3VTmvDRyAHcaT92whREFpLv9cj5lTeJSibyr/Mr
+m/YtjCZVWgaOYIhwrXwKLqPr/11inWsAkfIytvHWTxZYEcXLgAXFuUuaS3uF9gEi
+NQwzGTU1v0FqkqTBr4B8nW3HCN47XUu0t8Y0e+lf4s4OxQawWD79J9/5d3Ry0vbV
+3Am1FtGJiJvOwRsIfVChDpYStTcHTCMqtvWbV6L11BWkpzGXSW4Hv43qa+GSYOD2
+QU68Mb59oSk2OB+BtOLpJofmbGEGgvmwyCI9MwIDAQABAoIBACiARq2wkltjtcjs
+kFvZ7w1JAORHbEufEO1Eu27zOIlqbgyAcAl7q+/1bip4Z/x1IVES84/yTaM8p0go
+amMhvgry/mS8vNi1BN2SAZEnb/7xSxbflb70bX9RHLJqKnp5GZe2jexw+wyXlwaM
++bclUCrh9e1ltH7IvUrRrQnFJfh+is1fRon9Co9Li0GwoN0x0byrrngU8Ak3Y6D9
+D8GjQA4Elm94ST3izJv8iCOLSDBmzsPsXfcCUZfmTfZ5DbUDMbMxRnSo3nQeoKGC
+0Lj9FkWcfmLcpGlSXTO+Ww1L7EGq+PT3NtRae1FZPwjddQ1/4V905kyQFLamAA5Y
+lSpE2wkCgYEAy1OPLQcZt4NQnQzPz2SBJqQN2P5u3vXl+zNVKP8w4eBv0vWuJJF+
+hkGNnSxXQrTkvDOIUddSKOzHHgSg4nY6K02ecyT0PPm/UZvtRpWrnBjcEVtHEJNp
+bU9pLD5iZ0J9sbzPU/LxPmuAP2Bs8JmTn6aFRspFrP7W0s1Nmk2jsm0CgYEAyH0X
++jpoqxj4efZfkUrg5GbSEhf+dZglf0tTOA5bVg8IYwtmNk/pniLG/zI7c+GlTc9B
+BwfMr59EzBq/eFMI7+LgXaVUsM/sS4Ry+yeK6SJx/otIMWtDfqxsLD8CPMCRvecC
+2Pip4uSgrl0MOebl9XKp57GoaUWRWRHqwV4Y6h8CgYAZhI4mh4qZtnhKjY4TKDjx
+QYufXSdLAi9v3FxmvchDwOgn4L+PRVdMwDNms2bsL0m5uPn104EzM6w1vzz1zwKz
+5pTpPI0OjgWN13Tq8+PKvm/4Ga2MjgOgPWQkslulO/oMcXbPwWC3hcRdr9tcQtn9
+Imf9n2spL/6EDFId+Hp/7QKBgAqlWdiXsWckdE1Fn91/NGHsc8syKvjjk1onDcw0
+NvVi5vcba9oGdElJX3e9mxqUKMrw7msJJv1MX8LWyMQC5L6YNYHDfbPF1q5L4i8j
+8mRex97UVokJQRRA452V2vCO6S5ETgpnad36de3MUxHgCOX3qL382Qx9/THVmbma
+3YfRAoGAUxL/Eu5yvMK8SAt/dJK6FedngcM3JEFNplmtLYVLWhkIlNRGDwkg3I5K
+y18Ae9n7dHVueyslrb6weq7dTkYDi3iOYRW8HRkIQh06wEdbxt0shTzAJvvCQfrB
+jg/3747WSsf/zBTcHihTRBdAv6OmdhV4/dD5YBfLAkLrd+mX7iE=
+-----END RSA PRIVATE KEY-----

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/jwt_public.pem

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnzyis1ZjfNB0bBgKFMSv
+vkTtwlvBsaJq7S5wA+kzeVOVpVWwkWdVha4s38XM/pa/yr47av7+z3VTmvDRyAHc
+aT92whREFpLv9cj5lTeJSibyr/Mrm/YtjCZVWgaOYIhwrXwKLqPr/11inWsAkfIy
+tvHWTxZYEcXLgAXFuUuaS3uF9gEiNQwzGTU1v0FqkqTBr4B8nW3HCN47XUu0t8Y0
+e+lf4s4OxQawWD79J9/5d3Ry0vbV3Am1FtGJiJvOwRsIfVChDpYStTcHTCMqtvWb
+V6L11BWkpzGXSW4Hv43qa+GSYOD2QU68Mb59oSk2OB+BtOLpJofmbGEGgvmwyCI9
+MwIDAQAB
+-----END PUBLIC KEY-----

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/token.jwt

@@ -0,0 +1 @@
+eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJ0ZXN0Iiwic3ViIjoiaGFzaGlfdmF1bHRAdGVzdC5hbnNpYmxlLmNvbSIsIm5iZiI6MTYwNDgzNTEwMCwiZXhwIjozMjQ5OTA1MTM1OX0.NEWQR_Eicw8Fa9gU9HPY2M9Rp1czNTUKrICwKe7l1edaZNtgxhMGdyqnBsPrHL_dw1ZIwdvwVAioi8bEyIDEWICls0lzHwM169rrea3WEFrB5CP17A6DkvYL0cnOnGutbwUrXInPCRUfvRogIKEI-w8X-ris9LX2FBPKhXX1K3U0D8uYi5_9t8YWywTe0NkYvY-nTzMugK1MXMoBJ3fCksweJiDp6BOo3v9OU03MLgwgri2UdsqVb7WSk4XvWG-lmbiiSAWVf9BI3mecVDUHpYxbEqjv1HDG_wdX8zy1ZlAFbjp3kIpMlDVK1Q5nu_VPDzQrEvPdTnOzU36LE4UF-w

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/files/token_invalid.jwt

@@ -0,0 +1 @@
+eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMjM0IiwidXNlcl9jbGFpbSI6InVzZXJfY2xhaW0iLCJuYmYiOjE2MDQ4MzUxMDAsImV4cCI6MzI0OTkwNTEzNTl9.etc2WSH7kR3fHFlVt4wlBYFKNn7Z4DQcRVXUK4gGF-Q

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_setup.yml

@@ -1,7 +1,7 @@
 - name: 'Create an approle policy'
-  shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write approle-policy -"
+  command:
-  vars:
+    cmd: '{{ vault_cmd }} policy write approle-policy -'
-    policy: |
+    stdin: |
       path "auth/approle/login" {
        capabilities = [ "create", "read" ]
       }

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/jwt_setup.yml

@@ -0,0 +1,18 @@
+- name: 'Enable the JWT auth method'
+  command: '{{ vault_cmd }} auth enable jwt'
+
+- name: 'Configure the JWT auth method'
+  command: '{{ vault_cmd }} write auth/jwt/config jwt_validation_pubkeys={{ jwt_public_key | quote }}'
+  vars:
+    jwt_public_key: '{{ lookup("file", "jwt_public.pem") }}'
+
+- name: 'Create a named role'
+  command:
+    cmd: '{{ vault_cmd }} write auth/jwt/role/test-role -'
+    stdin: |
+      {
+        "role_type": "jwt",
+        "policies": "test-policy",
+        "user_claim": "sub",
+        "bound_audiences": "test"
+      }

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/jwt_test.yml

@@ -0,0 +1,46 @@
+- vars:
+    role_id: test-role
+    jwt: '{{ lookup("file", "token.jwt") }}'
+    jwt_invalid: '{{ lookup("file", "token_invalid.jwt") }}'
+  block:
+    - name: 'Fetch secrets using "hashi_vault" lookup'
+      set_fact:
+        secret1: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret1 auth_method=jwt jwt=' ~ jwt ~ ' role_id=' ~ role_id) }}"
+        secret2: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=jwt jwt=' ~ jwt ~ ' role_id=' ~ role_id) }}"
+
+    - name: 'Check secret values'
+      fail:
+        msg: 'unexpected secret values'
+      when: secret1['value'] != 'foo1' or secret2['value'] != 'foo2'
+
+    - name: 'Failure expected when erroneous credentials are used'
+      vars:
+        secret_wrong_cred: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=jwt jwt=' ~ jwt_invalid ~ ' role_id=' ~ role_id) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_wrong_cred }})'
+      register: test_wrong_cred
+      ignore_errors: true
+
+    - name: 'Failure expected when unauthorized secret is read'
+      vars:
+        secret_unauthorized: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret3 auth_method=jwt jwt=' ~ jwt ~ ' role_id=' ~ role_id) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_unauthorized }})'
+      register: test_unauthorized
+      ignore_errors: true
+
+    - name: 'Failure expected when non-existent secret is read'
+      vars:
+          secret_inexistent: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/non_existent_secret4 auth_method=jwt jwt=' ~ jwt ~ ' role_id=' ~ role_id) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_inexistent }})'
+      register: test_inexistent
+      ignore_errors: true
+
+    - name: 'Check expected failures'
+      assert:
+        msg: "an expected failure didn't occur"
+        that:
+          - test_wrong_cred is failed
+          - test_unauthorized is failed
+          - test_inexistent is failed

tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml

@@ -87,9 +87,9 @@
               command: '{{ vault_cmd }} secrets enable -path=kv2 -version=2 kv'
 
             - name: 'Create a test policy'
-              shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write test-policy -"
+              command:
-              vars:
+                cmd: '{{ vault_cmd }} policy write test-policy -'
-                policy: |
+                stdin: |
                   path "{{ vault_gen_path }}/secret1" {
                     capabilities = ["read"]
                   }
@@ -149,6 +149,10 @@
             - name: setup token auth
               import_tasks: token_setup.yml
 
+            - name: setup jwt auth
+              import_tasks: jwt_setup.yml
+              when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>')
+
         - import_tasks: tests.yml
           vars:
             auth_type: approle
@@ -158,6 +162,11 @@
           vars:
             auth_type: token
 
+        - import_tasks: tests.yml
+          vars:
+            auth_type: jwt
+          when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>')
+
       always:
         - name: 'Kill vault process'
           shell: "kill $(cat {{ local_temp_dir }}/vault.pid)"

tests/integration/targets/lookup_hashi_vault/playbooks/install_dependencies.yml

@@ -4,7 +4,7 @@
       import_role:
         name: setup_openssl
 
-    - name: "RedHat <= 7, select last version compatible with request 2.6.0 (this version doesn't support approle auth)"
+    - name: "RedHat <= 7, select last version compatible with request 2.6.0 (this version doesn't support approle or jwt auth)"
       set_fact:
         hvac_package: 'hvac==0.2.5'
       when: ansible_distribution == 'RedHat' and ansible_distribution_major_version is version('7', '<=')