diff --git a/plugins/lookup/etcd3.py b/plugins/lookup/etcd3.py new file mode 100644 index 0000000000..6f409434d6 --- /dev/null +++ b/plugins/lookup/etcd3.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +# +# (c) 2020, SCC France, Eric Belhomme +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = ''' + author: + - Eric Belhomme + lookup: etcd3 + short_description: Get key values from etcd3 server + description: + - Retrieves key values and/or key prefixes from etcd3 server using its native gRPC API. + - Try to reuse M(etcd3) options for connection parameters, but add support for some C(ETCDCTL_*) environment variables. + - See U(https://github.com/etcd-io/etcd/tree/master/Documentation/op-guide) for etcd overview. + + options: + _terms: + description: + - The list of keys (or key prefixes) to look up on the etcd3 server. + type: list + elements: str + required: True + prefix: + description: + - Look for key or prefix key. + type: bool + default: False + endpoints: + description: + - Counterpart of C(ETCDCTL_ENDPOINTS) enviroment variable. + Specify the etcd3 connection with and URL form eg. C(https://hostname:2379) or C(:) form. + - The C(host) part is overwritten by I(host) option, if defined. + - The C(port) part is overwritten by I(port) option, if defined. + env: + - name: ETCDCTL_ENDPOINTS + default: '127.0.0.1:2379' + type: str + host: + description: + - etcd3 listening client host. + - Takes precedence over I(endpoints). + type: str + port: + description: + - etcd3 listening client port. + - Takes precedence over I(endpoints). + type: int + ca_cert: + description: + - etcd3 CA authority. + env: + - name: ETCDCTL_CACERT + type: str + cert_cert: + description: + - etcd3 client certificate. + env: + - name: ETCDCTL_CERT + type: str + cert_key: + description: + - etcd3 client private key. + env: + - name: ETCDCTL_KEY + type: str + timeout: + description: + - Client timeout. + default: 60 + env: + - name: ETCDCTL_DIAL_TIMEOUT + type: int + user: + description: + - Authentified user name. + env: + - name: ETCDCTL_USER + type: str + password: + description: + - Authentified user password. + env: + - name: ETCDCTL_PASSWORD + type: str + + notes: + - I(host) and I(port) options take precedence over (endpoints) option. + - The recommanded way to connect to etcd3 server is using C(ETCDCTL_ENDPOINT) + environment variable and keep I(endpoints), I(host), and I(port) unused. + seealso: + - module: etcd3 + - ref: etcd_lookup + + requirements: + - "etcd3 >= 0.10" +''' + +EXAMPLES = ''' + - name: "a value from a locally running etcd" + debug: + msg: "{{ lookup('community.general.etcd3', 'foo/bar') }}" + + - name: "values from multiple folders on a locally running etcd" + debug: + msg: "{{ lookup('community.general.etcd3', 'foo', 'bar', 'baz') }}" + + - name: "look for a key prefix" + debug: + msg: "{{ lookup('community.general.etcd3', '/foo/bar', prefix=True) }}" + + - name: "connect to etcd3 with a client certificate" + debug: + msg: "{{ lookup('community.general.etcd3', 'foo/bar', cert_cert='/etc/ssl/etcd/client.pem', cert_key='/etc/ssl/etcd/client.key') }}" +''' + +RETURN = ''' + _raw: + description: + - List of keys and associated values. + type: list + elements: dict + contains: + key: + description: The element's key. + type: str + value: + description: The element's value. + type: str +''' + +import re + +from ansible.plugins.lookup import LookupBase +from ansible.utils.display import Display +from ansible.module_utils.basic import missing_required_lib +from ansible.module_utils._text import to_native +from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleError, AnsibleLookupError + +try: + import etcd3 + HAS_ETCD = True +except ImportError: + HAS_ETCD = False + +display = Display() + +etcd3_cnx_opts = ( + 'host', + 'port', + 'ca_cert', + 'cert_key', + 'cert_cert', + 'timeout', + 'user', + 'password', + # 'grpc_options' Etcd3Client() option currently not supported by lookup module (maybe in future ?) +) + + +def etcd3_client(client_params): + try: + etcd = etcd3.client(**client_params) + etcd.status() + except Exception as exp: + raise AnsibleLookupError('Cannot connect to etcd cluster: %s' % (to_native(exp))) + return etcd + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + self.set_options(var_options=variables, direct=kwargs) + + if not HAS_ETCD: + display.error(missing_required_lib('etcd3')) + return None + + # create the etcd3 connection parameters dict to pass to etcd3 class + client_params = {} + + # etcd3 class expects host and port as connection parameters, so endpoints + # must be mangled a bit to fit in this scheme. + # so here we use a regex to extract server and port + match = re.compile( + r'^(https?://)?(?P(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|([-_\d\w\.]+))(:(?P\d{1,5}))?/?$' + ).match(self.get_option('endpoints')) + if match: + if match.group('host'): + client_params['host'] = match.group('host') + if match.group('port'): + client_params['port'] = match.group('port') + + for opt in etcd3_cnx_opts: + if self.get_option(opt): + client_params[opt] = self.get_option(opt) + + cnx_log = dict(client_params) + if 'password' in cnx_log: + cnx_log['password'] = '' + display.verbose("etcd3 connection parameters: %s" % cnx_log) + + # connect to etcd3 server + etcd = etcd3_client(client_params) + + ret = [] + # we can pass many keys to lookup + for term in terms: + if self.get_option('prefix'): + try: + for val, meta in etcd.get_prefix(term): + if val and meta: + ret.append({'key': to_native(meta.key), 'value': to_native(val)}) + except Exception as exp: + display.warning('Caught except during etcd3.get_prefix: %s' % (to_native(exp))) + else: + try: + val, meta = etcd.get(term) + if val and meta: + ret.append({'key': to_native(meta.key), 'value': to_native(val)}) + except Exception as exp: + display.warning('Caught except during etcd3.get: %s' % (to_native(exp))) + return ret diff --git a/tests/integration/targets/lookup_etcd3/aliases b/tests/integration/targets/lookup_etcd3/aliases new file mode 100644 index 0000000000..c2ea7b42d1 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/aliases @@ -0,0 +1,8 @@ +shippable/posix/group1 +destructive +needs/file/tests/utils/constraints.txt +needs/target/setup_etcd3 +skip/aix +skip/osx +skip/freebsd +skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller diff --git a/tests/integration/targets/lookup_etcd3/defaults/main.yml b/tests/integration/targets/lookup_etcd3/defaults/main.yml new file mode 100644 index 0000000000..331ec312e7 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/defaults/main.yml @@ -0,0 +1,4 @@ +--- + + etcd3_prefix: '/keyprefix/' + etcd3_singlekey: '/singlekeypath' diff --git a/tests/integration/targets/lookup_etcd3/dependencies.yml b/tests/integration/targets/lookup_etcd3/dependencies.yml new file mode 100644 index 0000000000..e42f33badb --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/dependencies.yml @@ -0,0 +1,6 @@ +--- +- hosts: localhost + tasks: + - name: Setup etcd3 + import_role: + name: setup_etcd3 diff --git a/tests/integration/targets/lookup_etcd3/meta/main.yml b/tests/integration/targets/lookup_etcd3/meta/main.yml new file mode 100644 index 0000000000..5438ced5c3 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - setup_pkg_mgr diff --git a/tests/integration/targets/lookup_etcd3/runme.sh b/tests/integration/targets/lookup_etcd3/runme.sh new file mode 100755 index 0000000000..962201ffb3 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/runme.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +set -eux + +ANSIBLE_ROLES_PATH=../ \ + ansible-playbook dependencies.yml -v "$@" + +ANSIBLE_ROLES_PATH=../ \ + ansible-playbook test_lookup_etcd3.yml -v "$@" diff --git a/tests/integration/targets/lookup_etcd3/tasks/main.yml b/tests/integration/targets/lookup_etcd3/tasks/main.yml new file mode 100644 index 0000000000..d5ea58a305 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# lookup_etcd3 integration tests +# 2020, SCC France, Eric Belhomme +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +- name: put key/values with an etcd prefix + etcd3: + key: "{{ etcd3_prefix }}foo{{ item }}" + value: "bar{{ item }}" + state: present + loop: + - 1 + - 2 + - 3 + +- name: put a single key/values in etcd + etcd3: + key: "{{ etcd3_singlekey }}" + value: "foobar" + state: present + +- import_tasks: tests.yml diff --git a/tests/integration/targets/lookup_etcd3/tasks/tests.yml b/tests/integration/targets/lookup_etcd3/tasks/tests.yml new file mode 100644 index 0000000000..a1090b4809 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/tasks/tests.yml @@ -0,0 +1,26 @@ +--- +# lookup_etcd3 integration tests +# 2020, SCC France, Eric Belhomme +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +- block: + - name: 'Fetch secrets using "etcd3" lookup' + set_fact: + etcdoutkey1: "{{ lookup('community.general.etcd3', etcd3_prefix, prefix=True) }}" + etcdoutkey2: "{{ lookup('community.general.etcd3', etcd3_singlekey) }}" + key_inexistent: "{{ lookup('community.general.etcd3', 'inexistent_key') }}" + + - name: 'Check etcd values' + assert: + msg: 'unexpected etcd3 values' + that: + - etcdoutkey1 is sequence + - etcdoutkey1 | length() == 3 + - etcdoutkey1[0].value == 'bar1' + - etcdoutkey1[1].value == 'bar2' + - etcdoutkey1[2].value == 'bar3' + - etcdoutkey2 is sequence + - etcdoutkey2 | length() == 2 + - etcdoutkey2.value == 'foobar' + - key_inexistent is sequence + - key_inexistent | length() == 0 diff --git a/tests/integration/targets/lookup_etcd3/test_lookup_etcd3.yml b/tests/integration/targets/lookup_etcd3/test_lookup_etcd3.yml new file mode 100644 index 0000000000..583f2a6a08 --- /dev/null +++ b/tests/integration/targets/lookup_etcd3/test_lookup_etcd3.yml @@ -0,0 +1,6 @@ +--- +- hosts: localhost + tasks: + - name: Test lookup etcd3 + import_role: + name: lookup_etcd3 diff --git a/tests/integration/targets/setup_etcd3/defaults/main.yml b/tests/integration/targets/setup_etcd3/defaults/main.yml new file mode 100644 index 0000000000..95e4b837a7 --- /dev/null +++ b/tests/integration/targets/setup_etcd3/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# setup etcd3 for integration tests on module/lookup +# (c) 2017, Jean-Philippe Evrard +# 2020, SCC France, Eric Belhomme +# +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# # Copyright: (c) 2018, Ansible Project +# +etcd3_ver: "v3.2.14" +etcd3_download_server: "https://storage.googleapis.com/etcd" +#etcd3_download_server: "https://github.com/coreos/etcd/releases/download" +etcd3_download_url: "{{ etcd3_download_server }}/{{ etcd3_ver }}/etcd-{{ etcd3_ver }}-linux-amd64.tar.gz" +etcd3_download_location: /tmp/etcd-download-test +etcd3_path: "{{ etcd3_download_location }}/etcd-{{ etcd3_ver }}-linux-amd64" + +etcd3_pip_module: etcd3>=0.12 diff --git a/tests/integration/targets/setup_etcd3/meta/main.yml b/tests/integration/targets/setup_etcd3/meta/main.yml new file mode 100644 index 0000000000..5438ced5c3 --- /dev/null +++ b/tests/integration/targets/setup_etcd3/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - setup_pkg_mgr diff --git a/tests/integration/targets/setup_etcd3/tasks/main.yml b/tests/integration/targets/setup_etcd3/tasks/main.yml new file mode 100644 index 0000000000..d232571a0f --- /dev/null +++ b/tests/integration/targets/setup_etcd3/tasks/main.yml @@ -0,0 +1,112 @@ +--- +# setup etcd3 for integration tests on module/lookup +# (c) 2017, Jean-Philippe Evrard +# 2020, SCC France, Eric Belhomme + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# ============================================================ + +# setup etcd3 for supported distros +- block: + + - name: python 2 + set_fact: + python_suffix: "" + when: ansible_python_version is version('3', '<') + + - name: python 3 + set_fact: + python_suffix: "-py3" + when: ansible_python_version is version('3', '>=') + + - include_vars: '{{ item }}' + with_first_found: + - files: + - '{{ ansible_distribution }}-{{ ansible_distribution_major_version }}{{ python_suffix }}.yml' + - '{{ ansible_distribution }}-{{ ansible_distribution_version }}{{ python_suffix }}.yml' + - '{{ ansible_os_family }}-{{ ansible_distribution_major_version }}{{ python_suffix }}.yml' + - '{{ ansible_os_family }}{{ python_suffix }}.yml' + - 'default{{ python_suffix }}.yml' + - 'default.yml' + paths: '../vars' + + - name: Upgrade setuptools python2 module + pip: + name: setuptools<45 + extra_args: --upgrade + state: present + when: python_suffix == '' + + - name: Install etcd3 python modules + pip: + name: "{{ etcd3_pip_module }}" + extra_args: --only-binary grpcio + state: present + + # Check if re-installing etcd3 is required + - name: Check if etcd3ctl exists for re-use. + shell: "ETCDCTL_API=3 {{ etcd3_path }}/etcdctl --endpoints=localhost:2379 get foo" + args: + executable: /bin/bash + changed_when: false + failed_when: false + register: _testetcd3ctl + + - block: + # Installing etcd3 + - name: If can't reuse, prepare download folder + file: + path: "{{ etcd3_download_location }}" + state: directory + register: _etcddownloadexists + when: + - _testetcd3ctl.rc != 0 + + - name: Delete download folder if already exists (to start clean) + file: + path: "{{ etcd3_download_location }}" + state: absent + when: + - _etcddownloadexists is not changed + + - name: Recreate download folder if purged + file: + path: "{{ etcd3_download_location }}" + state: directory + when: + - _etcddownloadexists is not changed + + - name: Download etcd3 + unarchive: + src: "{{ etcd3_download_url }}" + dest: "{{ etcd3_download_location }}" + remote_src: yes + + # Running etcd3 and kill afterwards if it wasn't running before. + - name: Run etcd3 + shell: "{{ etcd3_path }}/etcd &" + register: _etcd3run + changed_when: true + +# - name: kill etcd3 +# command: "pkill etcd" + + when: + - _testetcd3ctl.rc != 0 + + when: + - ansible_distribution | lower ~ "-" ~ ansible_distribution_major_version | lower != 'centos-6' diff --git a/tests/integration/targets/setup_etcd3/vars/RedHat-7.yml b/tests/integration/targets/setup_etcd3/vars/RedHat-7.yml new file mode 100644 index 0000000000..2e0c082446 --- /dev/null +++ b/tests/integration/targets/setup_etcd3/vars/RedHat-7.yml @@ -0,0 +1 @@ +etcd3_pip_module: etcd3<0.12 \ No newline at end of file diff --git a/tests/integration/targets/setup_etcd3/vars/Suse-py3.yml b/tests/integration/targets/setup_etcd3/vars/Suse-py3.yml new file mode 100644 index 0000000000..bacd4a371a --- /dev/null +++ b/tests/integration/targets/setup_etcd3/vars/Suse-py3.yml @@ -0,0 +1,3 @@ +# SuSE's python 3.6.10 comes with six 1.11.0 as distutil +# we restrict to etcd3 < 0.11 to avoid pip to try to upgrade six +etcd3_pip_module: 'etcd3<0.11' diff --git a/tests/integration/targets/setup_etcd3/vars/Suse.yml b/tests/integration/targets/setup_etcd3/vars/Suse.yml new file mode 100644 index 0000000000..bacd4a371a --- /dev/null +++ b/tests/integration/targets/setup_etcd3/vars/Suse.yml @@ -0,0 +1,3 @@ +# SuSE's python 3.6.10 comes with six 1.11.0 as distutil +# we restrict to etcd3 < 0.11 to avoid pip to try to upgrade six +etcd3_pip_module: 'etcd3<0.11' diff --git a/tests/integration/targets/setup_etcd3/vars/default.yml b/tests/integration/targets/setup_etcd3/vars/default.yml new file mode 100644 index 0000000000..4a01b0ace8 --- /dev/null +++ b/tests/integration/targets/setup_etcd3/vars/default.yml @@ -0,0 +1,2 @@ +--- +# default should don't touch anything \ No newline at end of file diff --git a/tests/unit/plugins/lookup/test_etcd3.py b/tests/unit/plugins/lookup/test_etcd3.py new file mode 100644 index 0000000000..b0663dff66 --- /dev/null +++ b/tests/unit/plugins/lookup/test_etcd3.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# (c) 2020, SCC France, Eric Belhomme +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +import pytest +from ansible_collections.community.general.tests.unit.compat import unittest +from ansible_collections.community.general.tests.unit.compat.mock import patch, MagicMock +from ansible.errors import AnsibleError +from ansible_collections.community.general.plugins.lookup import etcd3 +from ansible.plugins.loader import lookup_loader + + +class FakeKVMetadata: + + def __init__(self, keyvalue, header): + self.key = keyvalue + self.create_revision = '' + self.mod_revision = '' + self.version = '' + self.lease_id = '' + self.response_header = header + + +class FakeEtcd3Client(MagicMock): + + def get_prefix(self, key): + for i in range(1, 4): + yield self.get('{0}_{1}'.format(key, i)) + + def get(self, key): + return ("{0} value".format(key), FakeKVMetadata(key, None)) + + +class TestLookupModule(unittest.TestCase): + + def setUp(self): + etcd3.HAS_ETCD = True + self.lookup = lookup_loader.get('community.general.etcd3') + + @patch('ansible_collections.community.general.plugins.lookup.etcd3.etcd3_client', FakeEtcd3Client()) + def test_key(self): + expected_result = [{'key': 'a_key', 'value': 'a_key value'}] + self.assertListEqual(expected_result, self.lookup.run(['a_key'], [])) + + @patch('ansible_collections.community.general.plugins.lookup.etcd3.etcd3_client', FakeEtcd3Client()) + def test_key_prefix(self): + expected_result = [ + {'key': 'a_key_1', 'value': 'a_key_1 value'}, + {'key': 'a_key_2', 'value': 'a_key_2 value'}, + {'key': 'a_key_3', 'value': 'a_key_3 value'}, + ] + self.assertListEqual(expected_result, self.lookup.run(['a_key'], [], **{'prefix': True}))