From eaa484eb37e244de4c39acf201b6d80eeed18acb Mon Sep 17 00:00:00 2001 From: Brian Scholer <1260690+briantist@users.noreply.github.com> Date: Tue, 28 Apr 2020 07:27:37 -0400 Subject: [PATCH] hashi_vault refresh - Add AWS login methods, bugfixes, cleanup (#23) * hashi_vault refresh from PR in ansible/ansible/#66735 * Duplicate AWS doc fragments, remove version_added * Restore FQCNames * Fully qualify examples * Add changelog for #23 hash_vault refresh * Reduce examples below 160 chars * Address review feedback * Update changelogs/fragments/23-hashi-vault-lookup-refresh.yaml Use review suggestion Co-Authored-By: flowerysong Co-authored-by: flowerysong --- .../23-hashi-vault-lookup-refresh.yaml | 13 + plugins/lookup/hashi_vault.py | 563 +++++++++++++----- .../lookup_hashi_vault/defaults/main.yml | 1 + .../lookup_hashi_vault/tasks/main.yml | 257 ++++---- .../lookup_hashi_vault/tasks/token_test.yml | 31 +- 5 files changed, 614 insertions(+), 251 deletions(-) create mode 100644 changelogs/fragments/23-hashi-vault-lookup-refresh.yaml diff --git a/changelogs/fragments/23-hashi-vault-lookup-refresh.yaml b/changelogs/fragments/23-hashi-vault-lookup-refresh.yaml new file mode 100644 index 0000000000..353d36bd19 --- /dev/null +++ b/changelogs/fragments/23-hashi-vault-lookup-refresh.yaml @@ -0,0 +1,13 @@ +bugfixes: + - hashi_vault - when a non-token authentication method like ldap or userpass failed, but a valid token was loaded anyway (via env or token file), the token was used to attempt authentication, hiding the failure of the requested auth method. + - hashi_vault - if used via ``with_hashi_vault`` and a list of n secrets to retrieve, only the first one would be retrieved and returned n times. + - hashi_vault - error messages are now user friendly and don't contain the secret name ( https://github.com/ansible-collections/community.general/issues/54 ) +minor_changes: + - hashi_vault - ``secret`` can now be an unnamed argument if it's specified first in the term string (see examples). + - hashi_vault - previously all options had to be supplied via key=value pairs in the term string; now a mix of string and parameters can be specified (see examples). + - hashi_vault - new option ``return_format`` added to control how secrets are returned, including options for multiple secrets and returning raw values with metadata. + - hashi_vault - ``token`` is now an explicit option (and the default) in the choices for ``auth_method``. This matches previous behavior (``auth_method`` omitted resulted in token auth) but makes the value clearer and allows it to be explicitly specified. + - hashi_vault - previous (undocumented) behavior was to attempt to read token from ``~/.vault-token`` if not specified. This is now controlled through ``token_path`` and ``token_file`` options (defaults will mimic previous behavior). + - hashi_vault - INI and additional ENV sources made available for some new and old options. + - hashi_vault - uses newer authentication calls in the HVAC library and falls back to older ones with deprecation warnings. + - hashi_vault - AWS IAM auth method added. Accepts standard ansible AWS params and only loads AWS libraries when needed. diff --git a/plugins/lookup/hashi_vault.py b/plugins/lookup/hashi_vault.py index e4a761574b..f93707651c 100644 --- a/plugins/lookup/hashi_vault.py +++ b/plugins/lookup/hashi_vault.py @@ -1,3 +1,4 @@ +# (c) 2020, Brian Scholer (@briantist) # (c) 2015, Jonathan Davila # (c) 2017 Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) @@ -5,105 +6,224 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -DOCUMENTATION = ''' +DOCUMENTATION = """ lookup: hashi_vault - author: Jonathan Davila - short_description: retrieve secrets from HashiCorp's vault + author: + - Jonathan Davila + - Brian Scholer (@briantist) + short_description: Retrieve secrets from HashiCorp's vault requirements: - hvac (python library) + - hvac 0.7.0+ (for namespace support) + - hvac 0.9.6+ (to avoid all deprecation warnings) + - botocore (only if inferring aws params from boto) + - boto3 (only if using a boto profile) description: - - retrieve secrets from HashiCorp's vault + - Retrieve secrets from HashiCorp's vault. notes: - Due to a current limitation in the HVAC library there won't necessarily be an error if a bad endpoint is specified. - - As of Ansible 2.10, only the latest secret is returned when specifying a KV v2 path. + - As of Ansible 2.10, only the latest version of a secret is returned when specifying a KV v2 path. + - As of Ansible 2.10, all options can be supplied via term string (space delimited key=value pairs) or by parameters (see examples). + - As of Ansible 2.10, when C(secret) is the first option in the term string, C(secret=) is not required (see examples). options: secret: description: query you are making. required: True token: - description: vault token. + description: + - Vault token. If using token auth and no token is supplied, explicitly or through env, then the plugin will check + - for a token file, as determined by C(token_path) and C(token_file). env: - name: VAULT_TOKEN + token_path: + description: If no token is specified, will try to read the token file from this path. + env: + - name: HOME + ini: + - section: lookup_hashi_vault + key: token_path + token_file: + description: If no token is specified, will try to read the token from this file in C(token_path). + ini: + - section: lookup_hashi_vault + key: token_file + default: '.vault-token' url: description: URL to vault service. env: - name: VAULT_ADDR + ini: + - section: lookup_hashi_vault + key: url default: 'http://127.0.0.1:8200' username: description: Authentication user name. password: description: Authentication password. role_id: - description: Role id for a vault AppRole auth. + description: Vault Role ID. Used in approle and aws_iam_login auth methods. env: - name: VAULT_ROLE_ID + ini: + - section: lookup_hashi_vault + key: role_id secret_id: description: Secret id for a vault AppRole auth. env: - name: VAULT_SECRET_ID auth_method: description: - - Authentication method to be used. - - C(userpass) is added in version 2.8. + - Authentication method to be used. + - C(userpass) is added in Ansible 2.8. + - C(aws_iam_login) is added in Ansible 2.10. env: - name: VAULT_AUTH_METHOD + ini: + - section: lookup_hashi_vault + key: auth_method choices: + - token - userpass - ldap - approle + - aws_iam_login + default: token + return_format: + description: + - Controls how multiple key/value pairs in a path are treated on return. + - C(dict) returns a single dict containing the key/value pairs (same behavior as before Ansible 2.10). + - C(values) returns a list of all the values only. Use when you don't care about the keys. + - C(raw) returns the actual API result, which includes metadata and may have the data nested in other keys. + choices: + - dict + - values + - raw + default: dict + aliases: [ as ] mount_point: - description: vault mount point, only required if you have a custom mount point. - default: ldap + description: Vault mount point, only required if you have a custom mount point. ca_cert: - description: path to certificate to use for authentication. + description: Path to certificate to use for authentication. aliases: [ cacert ] validate_certs: - description: controls verification and validation of SSL certificates, mostly you only want to turn off with self signed ones. + description: Controls verification and validation of SSL certificates, mostly you only want to turn off with self signed ones. type: boolean default: True namespace: - description: namespace where secrets reside. requires HVAC 0.7.0+ and Vault 0.11+. -''' + description: Namespace where secrets reside. Requires HVAC 0.7.0+ and Vault 0.11+. + aws_profile: + description: The AWS profile + type: str + aliases: [ boto_profile ] + env: + - name: AWS_DEFAULT_PROFILE + - name: AWS_PROFILE + aws_access_key: + description: The AWS access key to use. + type: str + aliases: [ aws_access_key_id ] + env: + - name: EC2_ACCESS_KEY + - name: AWS_ACCESS_KEY + - name: AWS_ACCESS_KEY_ID + aws_secret_key: + description: The AWS secret key that corresponds to the access key. + type: str + aliases: [ aws_secret_access_key ] + env: + - name: EC2_SECRET_KEY + - name: AWS_SECRET_KEY + - name: AWS_SECRET_ACCESS_KEY + aws_security_token: + description: The AWS security token if using temporary access and secret keys. + type: str + env: + - name: EC2_SECURITY_TOKEN + - name: AWS_SESSION_TOKEN + - name: AWS_SECURITY_TOKEN + region: + description: The AWS region for which to create the connection. + type: str + env: + - name: EC2_REGION + - name: AWS_REGION +""" EXAMPLES = """ - debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200') }}" - name: Return all secrets from a path debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200') }}" - name: Vault that requires authentication via LDAP debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hello:value auth_method=ldap mount_point=ldap username=myuser password=mypas url=http://myvault:8200')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret/hello:value auth_method=ldap mount_point=ldap username=myuser password=mypas') }}" - name: Vault that requires authentication via username and password debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hello:value auth_method=userpass username=myuser password=mypas url=http://myvault:8200')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello:value auth_method=userpass username=myuser password=psw url=http://myvault:8200') }}" - name: Using an ssl vault debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hola:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=https://myvault:8200 validate_certs=False')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hola:value token=c975b780-d1be-8016-866b-01d0f9b688a5 validate_certs=False') }}" - name: using certificate auth debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hi:value token=xxxx-xxx-xxx url=https://myvault:8200 validate_certs=True cacert=/cacert/path/ca.pem')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret/hi:value token=xxxx url=https://myvault:8200 validate_certs=True cacert=/cacert/path/ca.pem') }}" - name: authenticate with a Vault app role debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hello:value auth_method=approle role_id=myroleid secret_id=mysecretid url=http://myvault:8200')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello:value auth_method=approle role_id=myroleid secret_id=mysecretid') }}" - name: Return all secrets from a path in a namespace debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200 namespace=teama/admins')}}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 namespace=teama/admins') }}" # When using KV v2 the PATH should include "data" between the secret engine mount and path (e.g. "secret/data/:path") # see: https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version - name: Return latest KV v2 secret from path debug: - msg: "{{ lookup('hashi_vault', 'secret=secret/data/hello token=my_vault_token url=http://myvault_url:8200') }}" + msg: "{{ lookup('community.general.hashi_vault', 'secret=secret/data/hello token=my_vault_token url=http://myvault_url:8200') }}" +# The following examples work in collection releases after Ansible 2.10 +- name: secret= is not required if secret is first + debug: + msg: "{{ lookup('community.general.hashi_vault', 'secret/data/hello token= url=http://myvault_url:8200') }}" + +- name: options can be specified as parameters rather than put in term string + debug: + msg: "{{ lookup('community.general.hashi_vault', 'secret/data/hello', token=my_token_var, url='http://myvault_url:8200') }}" + +# return_format (or its alias 'as') can control how secrets are returned to you +- name: return secrets as a dict (default) + set_fact: + my_secrets: "{{ lookup('community.general.hashi_vault', 'secret/data/manysecrets', token=my_token_var, url='http://myvault_url:8200') }}" +- debug: + msg: "{{ my_secrets['secret_key'] }}" +- debug: + msg: "Secret '{{ item.key }}' has value '{{ item.value }}'" + loop: "{{ my_secrets | dict2items }}" + +- name: return secrets as values only + debug: + msg: "A secret value: {{ item }}" + loop: "{{ query('community.general.hashi_vault', 'secret/data/manysecrets', token=my_token_var, url='http://myvault_url:8200', return_format='values') }}" + +- name: return raw secret from API, including metadata + set_fact: + my_secret: "{{ lookup('community.general.hashi_vault', 'secret/data/hello:value', token=my_token_var, url='http://myvault_url:8200', as='raw') }}" +- debug: + msg: "This is version {{ my_secret['metadata']['version'] }} of hello:value. The secret data is {{ my_secret['data']['data']['value'] }}" + +# AWS IAM authentication method +# uses Ansible standard AWS options + +- name: authenticate with aws_iam_login + debug: + msg: "{{ lookup('community.general.hashi_vault', 'secret/hello:value', auth_method='aws_iam_login' role_id='myroleid', profile=my_boto_profile) }}" """ RETURN = """ @@ -115,8 +235,8 @@ _raw: import os from ansible.errors import AnsibleError -from ansible.module_utils.parsing.convert_bool import boolean from ansible.plugins.lookup import LookupBase +from ansible.utils.display import Display HAS_HVAC = False try: @@ -125,80 +245,97 @@ try: except ImportError: HAS_HVAC = False +HAS_BOTOCORE = False +try: + # import boto3 + import botocore + HAS_BOTOCORE = True +except ImportError: + HAS_BOTOCORE = False -ANSIBLE_HASHI_VAULT_ADDR = 'http://127.0.0.1:8200' - -if os.getenv('VAULT_ADDR') is not None: - ANSIBLE_HASHI_VAULT_ADDR = os.environ['VAULT_ADDR'] +HAS_BOTO3 = False +try: + import boto3 + # import botocore + HAS_BOTO3 = True +except ImportError: + HAS_BOTO3 = False class HashiVault: + def get_options(self, *option_names, **kwargs): + ret = {} + include_falsey = kwargs.get('include_falsey', False) + for option in option_names: + val = self.options.get(option) + if val or include_falsey: + ret[option] = val + return ret + def __init__(self, **kwargs): + self.options = kwargs - self.url = kwargs.get('url', ANSIBLE_HASHI_VAULT_ADDR) - self.namespace = kwargs.get('namespace', None) - self.avail_auth_method = ['approle', 'userpass', 'ldap'] + # check early that auth method is actually available + self.auth_function = 'auth_' + self.options['auth_method'] + if not (hasattr(self, self.auth_function) and callable(getattr(self, self.auth_function))): + raise AnsibleError( + "Authentication method '%s' is not implemented. ('%s' member function not found)" % (self.options['auth_method'], self.auth_function) + ) - # split secret arg, which has format 'secret/hello:value' into secret='secret/hello' and secret_field='value' - s = kwargs.get('secret') - if s is None: - raise AnsibleError("No secret specified for hashi_vault lookup") + client_args = { + 'url': self.options['url'], + 'verify': self.options['ca_cert'] + } - s_f = s.rsplit(':', 1) - self.secret = s_f[0] - if len(s_f) >= 2: - self.secret_field = s_f[1] - else: - self.secret_field = '' + if self.options.get('namespace'): + client_args['namespace'] = self.options['namespace'] - self.verify = self.boolean_or_cacert(kwargs.get('validate_certs', True), kwargs.get('cacert', '')) + # this is the only auth_method-specific thing here, because if we're using a token, we need it now + if self.options['auth_method'] == 'token': + client_args['token'] = self.options.get('token') - # If a particular backend is asked for (and its method exists) we call it, otherwise drop through to using - # token auth. This means if a particular auth backend is requested and a token is also given, then we - # ignore the token and attempt authentication against the specified backend. + self.client = hvac.Client(**client_args) + + # Check for old version, before auth_methods class (added in 0.7.0): + # https://github.com/hvac/hvac/releases/tag/v0.7.0 # - # to enable a new auth backend, simply add a new 'def auth_' method below. + # hvac is moving auth methods into the auth_methods class + # which lives in the client.auth member. # - self.auth_method = kwargs.get('auth_method', os.environ.get('VAULT_AUTH_METHOD')) - self.verify = self.boolean_or_cacert(kwargs.get('validate_certs', True), kwargs.get('cacert', '')) - if self.auth_method and self.auth_method != 'token': - try: - if self.namespace is not None: - self.client = hvac.Client(url=self.url, verify=self.verify, namespace=self.namespace) - else: - self.client = hvac.Client(url=self.url, verify=self.verify) - # prefixing with auth_ to limit which methods can be accessed - getattr(self, 'auth_' + self.auth_method)(**kwargs) - except AttributeError: - raise AnsibleError("Authentication method '%s' not supported." - " Available options are %r" % (self.auth_method, self.avail_auth_method)) - else: - self.token = kwargs.get('token', os.environ.get('VAULT_TOKEN', None)) - if self.token is None and os.environ.get('HOME'): - token_filename = os.path.join( - os.environ.get('HOME'), - '.vault-token' - ) - if os.path.exists(token_filename): - with open(token_filename) as token_file: - self.token = token_file.read().strip() + # Attempting to find which backends were moved into the class when (this is primarily for warnings): + # 0.7.0 -- github, ldap, mfa, azure?, gcp + # 0.7.1 -- okta + # 0.8.0 -- kubernetes + # 0.9.0 -- azure?, radius + # 0.9.3 -- aws + # 0.9.6 -- userpass + self.hvac_has_auth_methods = hasattr(self.client, 'auth') - if self.token is None: - raise AnsibleError("No Vault Token specified") - - if self.namespace is not None: - self.client = hvac.Client(url=self.url, token=self.token, verify=self.verify, namespace=self.namespace) - else: - self.client = hvac.Client(url=self.url, token=self.token, verify=self.verify) - - if not self.client.is_authenticated(): - raise AnsibleError("Invalid Hashicorp Vault Token Specified for hashi_vault lookup") + # We've already checked to ensure a method exists for a particular auth_method, of the form: + # + # auth_ + # + def authenticate(self): + getattr(self, self.auth_function)() def get(self): - data = self.client.read(self.secret) + '''gets a secret. should always return a list''' + secret = self.options['secret'] + field = self.options['secret_field'] + return_as = self.options['return_format'] + + try: + data = self.client.read(secret) + except hvac.exceptions.Forbidden: + raise AnsibleError("Forbidden: Permission Denied to secret '%s'." % secret) + + if data is None: + raise AnsibleError("The secret '%s' doesn't seem to exist." % secret) + + if return_as == 'raw': + return [data] # Check response for KV v2 fields and flatten nested secret data. - # # https://vaultproject.io/api/secret/kv/kv-v2.html#sample-response-1 try: # sentinel field checks @@ -209,65 +346,65 @@ class HashiVault: except KeyError: pass - if data is None: - raise AnsibleError("The secret %s doesn't seem to exist for hashi_vault lookup" % self.secret) + if return_as == 'values': + return list(data['data'].values()) - if self.secret_field == '': - return data['data'] + # everything after here implements return_as == 'dict' + if not field: + return [data['data']] - if self.secret_field not in data['data']: - raise AnsibleError("The secret %s does not contain the field '%s'. for hashi_vault lookup" % (self.secret, self.secret_field)) + if field not in data['data']: + raise AnsibleError("The secret %s does not contain the field '%s'. for hashi_vault lookup" % (secret, field)) - return data['data'][self.secret_field] + return [data['data'][field]] - def check_params(self, **kwargs): - username = kwargs.get('username') - if username is None: - raise AnsibleError("Authentication method %s requires a username" % self.auth_method) + # begin auth implementation methods + # + # To add new backends, 3 things should be added: + # + # 1. Add a new validate_auth_ method to the LookupModule, which is responsible for validating + # that it has the necessary options and whatever else it needs. + # + # 2. Add a new auth_ method to this class. These implementations are faily minimal as they should + # already have everything they need. This is also the place to check for deprecated auth methods as hvac + # continues to move backends into the auth_methods class. + # + # 3. Update the avail_auth_methods list in the LookupModules auth_methods() method (for now this is static). + # + def auth_token(self): + if not self.client.is_authenticated(): + raise AnsibleError("Invalid Hashicorp Vault Token Specified for hashi_vault lookup.") - password = kwargs.get('password') - if password is None: - raise AnsibleError("Authentication method %s requires a password" % self.auth_method) - - mount_point = kwargs.get('mount_point') - - return username, password, mount_point - - def auth_userpass(self, **kwargs): - username, password, mount_point = self.check_params(**kwargs) - if mount_point is None: - mount_point = 'userpass' - - self.client.auth_userpass(username, password, mount_point=mount_point) - - def auth_ldap(self, **kwargs): - username, password, mount_point = self.check_params(**kwargs) - if mount_point is None: - mount_point = 'ldap' - - self.client.auth.ldap.login(username, password, mount_point=mount_point) - - def boolean_or_cacert(self, validate_certs, cacert): - validate_certs = boolean(validate_certs, strict=False) - '''' return a bool or cacert ''' - if validate_certs is True: - if cacert != '': - return cacert - else: - return True + def auth_userpass(self): + params = self.get_options('username', 'password', 'mount_point') + if self.hvac_has_auth_methods and hasattr(self.client.auth.userpass, 'login'): + self.client.auth.userpass.login(**params) else: - return False + Display().warning("HVAC should be updated to version 0.9.6 or higher. Deprecated method 'auth_userpass' will be used.") + self.client.auth_userpass(**params) - def auth_approle(self, **kwargs): - role_id = kwargs.get('role_id', os.environ.get('VAULT_ROLE_ID', None)) - if role_id is None: - raise AnsibleError("Authentication method app role requires a role_id") + def auth_ldap(self): + params = self.get_options('username', 'password', 'mount_point') +# not hasattr(self.client, 'auth') + if self.hvac_has_auth_methods and hasattr(self.client.auth.ldap, 'login'): + self.client.auth.ldap.login(**params) + else: + Display().warning("HVAC should be updated to version 0.7.0 or higher. Deprecated method 'auth_ldap' will be used.") + self.client.auth_ldap(**params) - secret_id = kwargs.get('secret_id', os.environ.get('VAULT_SECRET_ID', None)) - if secret_id is None: - raise AnsibleError("Authentication method app role requires a secret_id") + def auth_approle(self): + params = self.get_options('role_id', 'secret_id') + self.client.auth_approle(**params) - self.client.auth_approle(role_id, secret_id) + def auth_aws_iam_login(self): + params = self.options['iam_login_credentials'] + if self.hvac_has_auth_methods and hasattr(self.client.auth.aws, 'iam_login'): + self.client.auth.aws.iam_login(**params) + else: + Display().warning("HVAC should be updated to version 0.9.3 or higher. Deprecated method 'auth_aws_iam' will be used.") + self.client.auth_aws_iam(**params) + + # end auth implementation methods class LookupModule(LookupBase): @@ -275,26 +412,160 @@ class LookupModule(LookupBase): if not HAS_HVAC: raise AnsibleError("Please pip install hvac to use the hashi_vault lookup module.") - vault_args = terms[0].split() - vault_dict = {} ret = [] - for param in vault_args: + for term in terms: + opts = kwargs.copy() + opts.update(self.parse_term(term)) + self.set_options(direct=opts) + self.process_options() + # FUTURE: Create one object, authenticate once, and re-use it, + # for gets, for better use during with_ loops. + client = HashiVault(**self._options) + client.authenticate() + ret.extend(client.get()) + + return ret + + def parse_term(self, term): + '''parses a term string into options''' + param_dict = {} + + for i, param in enumerate(term.split()): try: key, value = param.split('=') except ValueError: - raise AnsibleError("hashi_vault lookup plugin needs key=value pairs, but received %s" % terms) - vault_dict[key] = value + if (i == 0): + # allow secret to be specified as value only if it's first + key = 'secret' + value = param + else: + raise AnsibleError("hashi_vault lookup plugin needs key=value pairs, but received %s" % term) + param_dict[key] = value + return param_dict - if 'ca_cert' in vault_dict.keys(): - vault_dict['cacert'] = vault_dict['ca_cert'] - vault_dict.pop('ca_cert', None) + def process_options(self): + '''performs deep validation and value loading for options''' - vault_conn = HashiVault(**vault_dict) + # ca_cert to verify + self.boolean_or_cacert() - for term in terms: - key = term.split()[0] - value = vault_conn.get() - ret.append(value) + # auth methods + self.auth_methods() - return ret + # secret field splitter + self.field_ops() + + # begin options processing methods + + def boolean_or_cacert(self): + # This is needed because of this (https://hvac.readthedocs.io/en/stable/source/hvac_v1.html): + # + # # verify (Union[bool,str]) - Either a boolean to indicate whether TLS verification should + # # be performed when sending requests to Vault, or a string pointing at the CA bundle to use for verification. + # + '''' return a bool or cacert ''' + ca_cert = self.get_option('ca_cert') + validate_certs = self.get_option('validate_certs') + + if not (validate_certs and ca_cert): + self.set_option('ca_cert', validate_certs) + + def field_ops(self): + # split secret and field + secret = self.get_option('secret') + + s_f = secret.rsplit(':', 1) + self.set_option('secret', s_f[0]) + if len(s_f) >= 2: + field = s_f[1] + else: + field = None + self.set_option('secret_field', field) + + def auth_methods(self): + # enforce and set the list of available auth methods + # TODO: can this be read from the choices: field in documentation? + avail_auth_methods = ['token', 'approle', 'userpass', 'ldap', 'aws_iam_login'] + self.set_option('avail_auth_methods', avail_auth_methods) + auth_method = self.get_option('auth_method') + + if auth_method not in avail_auth_methods: + raise AnsibleError( + "Authentication method '%s' not supported. Available options are %r" % (auth_method, avail_auth_methods) + ) + + # run validator if available + auth_validator = 'validate_auth_' + auth_method + if hasattr(self, auth_validator) and callable(getattr(self, auth_validator)): + getattr(self, auth_validator)(auth_method) + + # end options processing methods + + # begin auth method validators + + def validate_by_required_fields(self, auth_method, *field_names): + missing = [field for field in field_names if not self.get_option(field)] + + if missing: + raise AnsibleError("Authentication method %s requires options %r to be set, but these are missing: %r" % (auth_method, field_names, missing)) + + def validate_auth_userpass(self, auth_method): + self.validate_by_required_fields(auth_method, 'username', 'password') + + def validate_auth_ldap(self, auth_method): + self.validate_by_required_fields(auth_method, 'username', 'password') + + def validate_auth_approle(self, auth_method): + self.validate_by_required_fields(auth_method, 'role_id', 'secret_id') + + def validate_auth_token(self, auth_method): + if auth_method == 'token': + if not self.get_option('token') and self.get_option('token_path'): + token_filename = os.path.join( + self.get_option('token_path'), + self.get_option('token_file') + ) + if os.path.exists(token_filename): + with open(token_filename) as token_file: + self.set_option('token', token_file.read().strip()) + + if not self.get_option('token'): + raise AnsibleError("No Vault Token specified or discovered.") + + def validate_auth_aws_iam_login(self, auth_method): + params = { + 'access_key': self.get_option('aws_access_key'), + 'secret_key': self.get_option('aws_secret_key') + } + + if self.get_option('role_id'): + params['role'] = self.get_option('role_id') + + if self.get_option('region'): + params['region'] = self.get_option('region') + + if not (params['access_key'] and params['secret_key']): + profile = self.get_option('aws_profile') + if profile: + # try to load boto profile + if not HAS_BOTO3: + raise AnsibleError("boto3 is required for loading a boto profile.") + session_credentials = boto3.session.Session(profile_name=profile).get_credentials() + else: + # try to load from IAM credentials + if not HAS_BOTOCORE: + raise AnsibleError("botocore is required for loading IAM role credentials.") + session_credentials = botocore.session.get_session().get_credentials() + + if not session_credentials: + raise AnsibleError("No AWS credentials supplied or available.") + + params['access_key'] = session_credentials.access_key + params['secret_key'] = session_credentials.secret_key + if session_credentials.token: + params['session_token'] = session_credentials.token + + self.set_option('iam_login_credentials', params) + + # end auth method validators diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml b/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml index f1f6dd981d..73f6915ce6 100644 --- a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml @@ -2,3 +2,4 @@ vault_gen_path: 'gen/testproject' vault_kv1_path: 'kv1/testproject' vault_kv2_path: 'kv2/data/testproject' +vault_kv2_multi_path: 'kv2/data/testmulti' diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml b/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml index c4f01e3feb..c4011e9fab 100644 --- a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml +++ b/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml @@ -1,110 +1,159 @@ +--- - name: Install Hashi Vault on controlled node and test vars: - vault_version: 0.11.0 - vault_uri: https://ansible-ci-files.s3.amazonaws.com/test/integration/targets/lookup_hashi_vault/vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}.zip + vault_version: '0.11.0' + vault_uri: 'https://ansible-ci-files.s3.amazonaws.com/test/integration/targets/lookup_hashi_vault/vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}.zip' vault_cmd: '{{ local_temp_dir }}/vault' block: - - name: Create a local temporary directory - tempfile: - state: directory - register: tempfile_result - - set_fact: - local_temp_dir: '{{ tempfile_result.path }}' - - when: pyopenssl_version.stdout is version('0.15', '>=') - block: - - name: Generate privatekey - community.crypto.openssl_privatekey: - path: '{{ local_temp_dir }}/privatekey.pem' - - name: Generate CSR - community.crypto.openssl_csr: - path: '{{ local_temp_dir }}/csr.csr' - privatekey_path: '{{ local_temp_dir }}/privatekey.pem' - subject: - commonName: localhost - - name: Generate selfsigned certificate - register: selfsigned_certificate - community.crypto.openssl_certificate: - path: '{{ local_temp_dir }}/cert.pem' - csr_path: '{{ local_temp_dir }}/csr.csr' - privatekey_path: '{{ local_temp_dir }}/privatekey.pem' - provider: selfsigned - selfsigned_digest: sha256 - - name: Install unzip - package: - name: unzip - when: ansible_distribution != "MacOSX" - - assert: - that: ansible_architecture in ['i386', 'x86_64', 'amd64'] - - set_fact: - vault_arch: '386' - when: ansible_architecture == 'i386' - - set_fact: - vault_arch: amd64 - when: ansible_architecture in ['x86_64', 'amd64'] - - name: Download vault binary - unarchive: - src: '{{ vault_uri }}' - dest: '{{ local_temp_dir }}' - remote_src: true - - environment: - VAULT_DEV_ROOT_TOKEN_ID: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a - block: - - name: Create configuration file - template: - src: vault_config.hcl.j2 - dest: '{{ local_temp_dir }}/vault_config.hcl' - - name: Start vault service - environment: - VAULT_ADDR: http://localhost:8200 + - name: Create a local temporary directory + tempfile: + state: directory + register: tempfile_result + + - set_fact: + local_temp_dir: '{{ tempfile_result.path }}' + + - when: pyopenssl_version.stdout is version('0.15', '>=') block: - - name: Start vault server (dev mode enabled) - shell: nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 & - - name: Create generic secrets engine - command: '{{ vault_cmd }} secrets enable -path=gen generic' - - name: Create KV v1 secrets engine - command: '{{ vault_cmd }} secrets enable -path=kv1 -version=1 kv' - - name: Create KV v2 secrets engine - command: '{{ vault_cmd }} secrets enable -path=kv2 -version=2 kv' - - name: Create a test policy - shell: echo '{{ policy }}' | {{ vault_cmd }} policy write test-policy - - vars: - policy: "path \"{{ vault_gen_path }}/secret1\" {\n capabilities = [\"read\"]\n}\npath \"{{ vault_gen_path }}/secret2\" {\n capabilities = [\"read\", \"update\"]\n}\npath \"{{ vault_gen_path }}/secret3\" {\n capabilities = [\"deny\"]\n}\npath \"{{ vault_kv1_path }}/secret1\" {\n capabilities = [\"read\"]\n}\npath \"{{ vault_kv1_path }}/secret2\" {\n capabilities = [\"read\", \"update\"]\n}\npath \"{{ vault_kv1_path }}/secret3\" {\n capabilities = [\"deny\"]\n}\npath \"{{ vault_kv2_path }}/secret1\" {\n capabilities = [\"read\"]\n}\npath \"{{ vault_kv2_path }}/secret2\" {\n capabilities = [\"read\", \"update\"]\n}\npath \"{{ vault_kv2_path }}/secret3\" {\n capabilities = [\"deny\"]\n}\n" - - name: Create generic secrets - command: '{{ vault_cmd }} write {{ vault_gen_path }}/secret{{ item }} value=foo{{ item }}' - loop: - - 1 - - 2 - - 3 - - name: Create KV v1 secrets - command: '{{ vault_cmd }} kv put {{ vault_kv1_path }}/secret{{ item }} value=foo{{ item }}' - loop: - - 1 - - 2 - - 3 - - name: Create KV v2 secrets - command: '{{ vault_cmd }} kv put {{ vault_kv2_path | regex_replace("/data") }}/secret{{ item }} value=foo{{ item }}' - loop: - - 1 - - 2 - - 3 - - name: setup approle auth - import_tasks: approle_setup.yml - when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>') - - name: setup token auth - import_tasks: token_setup.yml - - import_tasks: tests.yml - vars: - auth_type: approle - when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>') - - import_tasks: tests.yml - vars: - auth_type: token - always: - - name: Kill vault process - shell: kill $(cat {{ local_temp_dir }}/vault.pid) - ignore_errors: true + - name: Generate privatekey + community.crypto.openssl_privatekey: + path: '{{ local_temp_dir }}/privatekey.pem' + + - name: Generate CSR + community.crypto.openssl_csr: + path: '{{ local_temp_dir }}/csr.csr' + privatekey_path: '{{ local_temp_dir }}/privatekey.pem' + subject: + commonName: localhost + + - name: Generate selfsigned certificate + community.crypto.openssl_certificate: + path: '{{ local_temp_dir }}/cert.pem' + csr_path: '{{ local_temp_dir }}/csr.csr' + privatekey_path: '{{ local_temp_dir }}/privatekey.pem' + provider: selfsigned + selfsigned_digest: sha256 + register: selfsigned_certificate + + - name: 'Install unzip' + package: + name: unzip + when: ansible_distribution != "MacOSX" # unzip already installed + + - assert: + # Linux: x86_64, FreeBSD: amd64 + that: ansible_architecture in ['i386', 'x86_64', 'amd64'] + - set_fact: + vault_arch: '386' + when: ansible_architecture == 'i386' + - set_fact: + vault_arch: amd64 + when: ansible_architecture in ['x86_64', 'amd64'] + + - name: 'Download vault binary' + unarchive: + src: '{{ vault_uri }}' + dest: '{{ local_temp_dir }}' + remote_src: true + + - environment: + # used by vault command + VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a' + block: + - name: 'Create configuration file' + template: + src: vault_config.hcl.j2 + dest: '{{ local_temp_dir }}/vault_config.hcl' + + - name: 'Start vault service' + environment: + VAULT_ADDR: 'http://localhost:8200' + block: + - name: 'Start vault server (dev mode enabled)' + shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' + + - name: 'Create generic secrets engine' + command: '{{ vault_cmd }} secrets enable -path=gen generic' + + - name: 'Create KV v1 secrets engine' + command: '{{ vault_cmd }} secrets enable -path=kv1 -version=1 kv' + + - name: 'Create KV v2 secrets engine' + command: '{{ vault_cmd }} secrets enable -path=kv2 -version=2 kv' + + - name: 'Create a test policy' + shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write test-policy -" + vars: + policy: | + path "{{ vault_gen_path }}/secret1" { + capabilities = ["read"] + } + path "{{ vault_gen_path }}/secret2" { + capabilities = ["read", "update"] + } + path "{{ vault_gen_path }}/secret3" { + capabilities = ["deny"] + } + path "{{ vault_kv1_path }}/secret1" { + capabilities = ["read"] + } + path "{{ vault_kv1_path }}/secret2" { + capabilities = ["read", "update"] + } + path "{{ vault_kv1_path }}/secret3" { + capabilities = ["deny"] + } + path "{{ vault_kv2_path }}/secret1" { + capabilities = ["read"] + } + path "{{ vault_kv2_path }}/secret2" { + capabilities = ["read", "update"] + } + path "{{ vault_kv2_path }}/secret3" { + capabilities = ["deny"] + } + path "{{ vault_kv2_multi_path }}/secrets" { + capabilities = ["read"] + } + + - name: 'Create generic secrets' + command: '{{ vault_cmd }} write {{ vault_gen_path }}/secret{{ item }} value=foo{{ item }}' + loop: [1, 2, 3] + + - name: 'Create KV v1 secrets' + command: '{{ vault_cmd }} kv put {{ vault_kv1_path }}/secret{{ item }} value=foo{{ item }}' + loop: [1, 2, 3] + + - name: 'Create KV v2 secrets' + command: '{{ vault_cmd }} kv put {{ vault_kv2_path | regex_replace("/data") }}/secret{{ item }} value=foo{{ item }}' + loop: [1, 2, 3] + + - name: 'Create multiple KV v2 secrets under one path' + command: '{{ vault_cmd }} kv put {{ vault_kv2_multi_path | regex_replace("/data") }}/secrets value1=foo1 value2=foo2 value3=foo3' + + - name: setup approle auth + import_tasks: approle_setup.yml + when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>') + + - name: setup token auth + import_tasks: token_setup.yml + + - import_tasks: tests.yml + vars: + auth_type: approle + when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>') + + - import_tasks: tests.yml + vars: + auth_type: token + + always: + - name: 'Kill vault process' + shell: "kill $(cat {{ local_temp_dir }}/vault.pid)" + ignore_errors: true + always: - - name: Delete temp dir - file: - path: '{{ local_temp_dir }}' - state: absent + - name: 'Delete temp dir' + file: + path: '{{ local_temp_dir }}' + state: absent diff --git a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml b/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml index 542284517e..55cb1a3e04 100644 --- a/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml +++ b/tests/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml @@ -9,6 +9,9 @@ kv1_secret2: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv1_path ~ '/secret2 token=' ~ user_token) }}" kv2_secret1: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret1 auth_method=token token=' ~ user_token) }}" kv2_secret2: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 token=' ~ user_token) }}" + kv2_secret2_as_raw: "{{ lookup('community.general.hashi_vault', vault_kv2_path ~ '/secret2 ' ~ conn_params, auth_method='token', token=user_token, return_format='raw') }}" + kv2_secrets_as_dict: "{{ lookup('community.general.hashi_vault', vault_kv2_multi_path ~ '/secrets ' ~ conn_params, auth_method='token', token=user_token) }}" + kv2_secrets_as_values: "{{ query('community.general.hashi_vault', vault_kv2_multi_path ~ '/secrets ' ~ conn_params, auth_method='token', token=user_token, return_format='values') }}" - name: 'Check secret generic values' fail: @@ -25,6 +28,32 @@ msg: 'unexpected secret values' when: kv2_secret1['value'] != 'foo1' or kv2_secret2['value'] != 'foo2' + - name: 'Check kv2 secret raw return value' + fail: + msg: + when: >- + 'data' not in kv2_secret2_as_raw + or 'data' not in kv2_secret2_as_raw['data'] + or 'metadata' not in kv2_secret2_as_raw['data'] + + - name: "Check multiple secrets as dict" + fail: + msg: 'Return value was not dict or items do not match.' + when: (kv2_secrets_as_dict | type_debug != 'dict') or (kv2_secrets_as_dict['value{{ item }}'] != 'foo{{ item }}') + loop: [1, 2, 3] + + - name: "Check multiple secrets as values" + fail: + msg: 'Return value was not list or items do not match.' + when: (kv2_secrets_as_values | type_debug != 'list') or ('foo{{ item }}' not in kv2_secrets_as_values) + loop: [1, 2, 3] + + - name: "Check multiple secrets as dict" + fail: + msg: 'Return value was not dict or items do not match.' + when: (kv2_secrets_as_dict | type_debug != 'dict') or (kv2_secrets_as_dict['value{{ item }}'] != 'foo{{ item }}') + loop: [1, 2, 3] + - name: 'Failure expected when erroneous credentials are used' vars: secret_wrong_cred: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=token token=wrong_token') }}" @@ -43,7 +72,7 @@ - name: 'Failure expected when inexistent secret is read' vars: - secret_inexistent: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret4 token=' ~ user_token) }}" + secret_inexistent: "{{ lookup('community.general.hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret4 token=' ~ user_token) }}" debug: msg: 'Failure is expected ({{ secret_inexistent }})' register: test_inexistent