From 1a13287788eaf10d0347b38b22980ac3aec2dc4d Mon Sep 17 00:00:00 2001 From: Mike Hume Date: Sat, 6 Jun 2020 12:36:28 -0700 Subject: [PATCH] Add sops lookup plugin (#374) * add sops lookup plugin * fix pylint * fix undefined encrypted_file variable * decode sops output as text by default * add variable to control decrypted content print in logs * use Sops class decryption method * lookup should return text, use appropriate ansible facility * use ansible.module_utils._text.to_native As required by Ansible documentation on [raising errors][raising-errors] from plugins, use to_native to wrap errors to ensure string compatibility between Python versions. [raising-errors]: https://docs.ansible.com/ansible/latest/dev_guide/developing_plugins.html#id3 * use with_items instead of with_file in sops lookup documentation [with_file][with-file], per Ansible documentation, returns the content of the file. As sops is not able to decrypt a string by itself but requires the file is passed as argument, passing the content breaks the lookup plugin as reported by [here][bug-report]. [with_items][with-items] should be used instead. [with-file]: https://docs.ansible.com/ansible/2.4/playbooks_loops.html#looping-over-files [with-items]: https://docs.ansible.com/ansible/2.4/playbooks_loops.html#standard-loops [bug-report]: https://github.com/ansible/ansible/pull/59639#issuecomment-540803722 * uniform sops exception handling between plugins * Apply suggestions from code review Co-Authored-By: Felix Fontein * remove sops lookup plugin print option Is no longer possible to print the decrypted secrets directly from this plugin, but `debug` module can be used instead. * add github handle to author * add setup_sops integration target * extract sops module * add lookup_sops integration tests * use sops module * Update plugins/module_utils/sops.py Co-authored-by: Felix Fontein * Update plugins/module_utils/sops.py Co-authored-by: Felix Fontein * Update plugins/lookup/sops.py Co-authored-by: Felix Fontein * Update plugins/module_utils/sops.py Co-authored-by: Felix Fontein * Update plugins/module_utils/sops.py Co-authored-by: Felix Fontein * Update test/integration/targets/lookup_sops/tasks/ubuntu.yml Co-authored-by: Felix Fontein * Update plugins/module_utils/sops.py Co-authored-by: Felix Fontein * Update test/integration/targets/lookup_sops/files/simple.sops.yaml Co-authored-by: Felix Fontein * Adding aliases file * Emtpy spaces * Update plugins/lookup/sops.py Co-authored-by: Felix Fontein * Update plugins/lookup/sops.py Co-authored-by: Felix Fontein * Update plugins/lookup/sops.py Co-authored-by: Felix Fontein * Update test/integration/targets/lookup_sops/tasks/ubuntu.yml Co-authored-by: Felix Fontein * Update plugins/lookup/sops.py Co-authored-by: Felix Fontein * Update test/integration/targets/lookup_sops/tasks/ubuntu.yml Co-authored-by: Felix Fontein * Update test/integration/targets/lookup_sops/tasks/ubuntu.yml Co-authored-by: Felix Fontein * gpg -> gnupg2 * with_items -> loop * Move error logic to module_utils. * Make Sops.decrypt() also handle errors and decode output. * Improve error handling. * Improve example formatting. * Reorganize tests. * Add test. * Remove version_added. Co-authored-by: Edoardo Tenani Co-authored-by: Edoardo Tenani Co-authored-by: Edoardo T Co-authored-by: Felix Fontein --- plugins/lookup/sops.py | 92 +++++++++++++++++++ plugins/module_utils/sops.py | 79 ++++++++++++++++ tests/integration/targets/lookup_sops/aliases | 5 + .../targets/lookup_sops/files/.sops.yaml | 3 + .../lookup_sops/files/simple.sops.yaml | 25 +++++ .../targets/lookup_sops/files/wrong.yaml | 2 + .../targets/lookup_sops/meta/main.yml | 3 + .../targets/lookup_sops/tasks/main.yml | 36 ++++++++ .../targets/setup_sops/meta/main.yml | 1 + .../targets/setup_sops/tasks/main.yml | 7 ++ .../targets/setup_sops/tasks/ubuntu.yml | 20 ++++ 11 files changed, 273 insertions(+) create mode 100644 plugins/lookup/sops.py create mode 100644 plugins/module_utils/sops.py create mode 100644 tests/integration/targets/lookup_sops/aliases create mode 100644 tests/integration/targets/lookup_sops/files/.sops.yaml create mode 100644 tests/integration/targets/lookup_sops/files/simple.sops.yaml create mode 100644 tests/integration/targets/lookup_sops/files/wrong.yaml create mode 100644 tests/integration/targets/lookup_sops/meta/main.yml create mode 100644 tests/integration/targets/lookup_sops/tasks/main.yml create mode 100644 tests/integration/targets/setup_sops/meta/main.yml create mode 100644 tests/integration/targets/setup_sops/tasks/main.yml create mode 100644 tests/integration/targets/setup_sops/tasks/ubuntu.yml diff --git a/plugins/lookup/sops.py b/plugins/lookup/sops.py new file mode 100644 index 0000000000..a6a5dea07a --- /dev/null +++ b/plugins/lookup/sops.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018 Edoardo Tenani [@endorama] +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleLookupError +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_native +from ansible_collections.community.general.plugins.module_utils.sops import Sops, SopsError + +from ansible.utils.display import Display +display = Display() + + +DOCUMENTATION = """ + lookup: sops + author: Edoardo Tenani (@endorama) + short_description: Read sops encrypted file contents + description: + - This lookup returns the contents from a file on the Ansible controller's file system. + - This lookup requires the C(sops) executable to be available in the controller PATH. + options: + _terms: + description: path(s) of files to read + required: True + notes: + - This lookup does not understand 'globbing' - use the fileglob lookup instead. +""" + +EXAMPLES = """ +tasks: + - name: Output secrets to screen (BAD IDEA!) + debug: + msg: "Content: {{ lookup('sops', item) }}" + loop: + - sops-encrypted-file.enc.yaml + + - name: Add SSH private key + copy: + content: "{{ lookup('sops', user + '-id_rsa') }}" + dest: /home/{{ user }}/.ssh/id_rsa + owner: "{{ user }}" + group: "{{ user }}" + mode: 0600 + no_log: true # avoid content to be written to log +""" + +RETURN = """ + _raw: + description: decrypted file content +""" + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + ret = [] + + for term in terms: + display.debug("Sops lookup term: %s" % term) + lookupfile = self.find_file_in_search_path(variables, 'files', term) + display.vvvv(u"Sops lookup using %s as file" % lookupfile) + + if not lookupfile: + raise AnsibleLookupError("could not locate file in lookup: %s" % to_native(term)) + + try: + output = Sops.decrypt(lookupfile, display=display) + except SopsError as e: + raise AnsibleLookupError(to_native(e)) + + ret.append(output.rstrip()) + + return ret diff --git a/plugins/module_utils/sops.py b/plugins/module_utils/sops.py new file mode 100644 index 0000000000..d967201208 --- /dev/null +++ b/plugins/module_utils/sops.py @@ -0,0 +1,79 @@ +# Copyright (c), Edoardo Tenani , 2018-2020 +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +from ansible.module_utils._text import to_text, to_native + +from subprocess import Popen, PIPE + + +# From https://github.com/mozilla/sops/blob/master/cmd/sops/codes/codes.go +# Should be manually updated +SOPS_ERROR_CODES = { + 1: "ErrorGeneric", + 2: "CouldNotReadInputFile", + 3: "CouldNotWriteOutputFile", + 4: "ErrorDumpingTree", + 5: "ErrorReadingConfig", + 6: "ErrorInvalidKMSEncryptionContextFormat", + 7: "ErrorInvalidSetFormat", + 8: "ErrorConflictingParameters", + 21: "ErrorEncryptingMac", + 23: "ErrorEncryptingTree", + 24: "ErrorDecryptingMac", + 25: "ErrorDecryptingTree", + 49: "CannotChangeKeysFromNonExistentFile", + 51: "MacMismatch", + 52: "MacNotFound", + 61: "ConfigFileNotFound", + 85: "KeyboardInterrupt", + 91: "InvalidTreePathFormat", + 100: "NoFileSpecified", + 128: "CouldNotRetrieveKey", + 111: "NoEncryptionKeyFound", + 200: "FileHasNotBeenModified", + 201: "NoEditorFound", + 202: "FailedToCompareVersions", + 203: "FileAlreadyEncrypted" +} + + +class SopsError(Exception): + ''' Extend Exception class with sops specific informations ''' + + def __init__(self, filename, exit_code, message): + if exit_code in SOPS_ERROR_CODES: + exception_name = SOPS_ERROR_CODES[exit_code] + message = "error with file %s: %s exited with code %d: %s" % (filename, exception_name, exit_code, to_native(message)) + else: + message = "could not decrypt file %s; Unknown sops error code: %s" % (filename, to_native(exit_code)) + super(SopsError, self).__init__(message) + + +class Sops(): + ''' Utility class to perform sops CLI actions ''' + + @staticmethod + def decrypt(encrypted_file, display=None): + # Run sops directly, python module is deprecated + command = ["sops", "--decrypt", encrypted_file] + process = Popen(command, stdout=PIPE, stderr=PIPE) + (output, err) = process.communicate() + exit_code = process.returncode + + # output is binary, we want UTF-8 string + output = to_text(output, errors='surrogate_or_strict') + # the process output is the decrypted secret; be cautious + + # sops logs always to stderr, as stdout is used for + # file content + if err and display: + display.vvvv(err) + + if exit_code > 0: + raise SopsError(encrypted_file, exit_code, err) + + return output.rstrip() diff --git a/tests/integration/targets/lookup_sops/aliases b/tests/integration/targets/lookup_sops/aliases new file mode 100644 index 0000000000..0f57a49343 --- /dev/null +++ b/tests/integration/targets/lookup_sops/aliases @@ -0,0 +1,5 @@ +shippable/posix/group1 +skip/aix +skip/osx +skip/freebsd +skip/python2.6 # lookups are controller only, and we no longer support Python 2.6 on the controller \ No newline at end of file diff --git a/tests/integration/targets/lookup_sops/files/.sops.yaml b/tests/integration/targets/lookup_sops/files/.sops.yaml new file mode 100644 index 0000000000..2ae15578a6 --- /dev/null +++ b/tests/integration/targets/lookup_sops/files/.sops.yaml @@ -0,0 +1,3 @@ +--- +creation_rules: + - pgp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4 diff --git a/tests/integration/targets/lookup_sops/files/simple.sops.yaml b/tests/integration/targets/lookup_sops/files/simple.sops.yaml new file mode 100644 index 0000000000..d7bfc2ea21 --- /dev/null +++ b/tests/integration/targets/lookup_sops/files/simple.sops.yaml @@ -0,0 +1,25 @@ +foo: ENC[AES256_GCM,data:a25L,iv:X8ILHZr+YiyLWa90Y+cwoMD1nVuel7JyTs0A5+oiOOo=,tag:GbBtp+Yqx1KEjdyztqS4EQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + lastmodified: '2020-02-20T10:44:32Z' + mac: ENC[AES256_GCM,data:BAwQqD9sHgHkmlxPQLKq28Xy48qPp1B/+GDLEsIxir6WNhZgw8OgjVF1u/wCAad6qHkmN02Bwenr+aay6uKfCuOEsTRSvZ7v80yAU+h0wL3zJ/KMkRsE3QP3CWxcLQxInt+YaBjR+Q0IUjDXKm3u6ZomixZe5F5pwWr36ErV6Y0=,iv:e/iiyXQiCh8C2w/bc8mr/Psv+ehmqEMqEC1/bbGFHpY=,tag:NSDo2HISIBJhYvsqrU0mSA==,type:str] + pgp: + - created_at: '2020-02-20T10:44:32Z' + enc: |- + -----BEGIN PGP MESSAGE----- + + wcBMAyUpShfNkFB/AQgALJTUwdx6rAPckJ+reP5TEq+lXzHI1Zi7aHYOqZQBnA2s + z8h1gRce/fn7RPkmdsjsdSYmxGGKqwDXxUYsbN1aWXk6mb4Juktdvjl/GndF6PkU + TiN/l1GM6upgS+GPxA01NKsGkVmEtKR5NhsNEnE6OzY29+PFLsBX2vO1Zfg7kzBz + cDl6PT8fbFTEaFeyuYl9IslIV8yYsj1oHL3CF76RjCP6b18NSOHM23ytlH+KVaBV + ntoSVkTyWDx5o9iEHBEWSEGNpaCWWiEgkDEkA1VqMHdUlsW+IjZ8ggg5NJbcVtrG + YkN8rlGsNEzx+g4O4b1160A2K6AdTBcoGHwHD3u3XdLgAeTqT1ekE2N3yNT6w4sm + 6uET4eTS4Cvg1OFCgOC34uUzlY3gbuVy20h8RNyQoAfhSN4DD2MexKqcMMCVCtn0 + OhRMTP2jjOCe5Ex3/p3awcVxwx7qeJ26Vnfiwtg6ueFI5AA= + =tcnq + -----END PGP MESSAGE----- + fp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4 + unencrypted_suffix: _unencrypted + version: 3.4.0 diff --git a/tests/integration/targets/lookup_sops/files/wrong.yaml b/tests/integration/targets/lookup_sops/files/wrong.yaml new file mode 100644 index 0000000000..2df127395d --- /dev/null +++ b/tests/integration/targets/lookup_sops/files/wrong.yaml @@ -0,0 +1,2 @@ +--- +this-is-not: a sops file diff --git a/tests/integration/targets/lookup_sops/meta/main.yml b/tests/integration/targets/lookup_sops/meta/main.yml new file mode 100644 index 0000000000..51b2487b2d --- /dev/null +++ b/tests/integration/targets/lookup_sops/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - setup_sops diff --git a/tests/integration/targets/lookup_sops/tasks/main.yml b/tests/integration/targets/lookup_sops/tasks/main.yml new file mode 100644 index 0000000000..17d9129cd4 --- /dev/null +++ b/tests/integration/targets/lookup_sops/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- when: sops_installed + block: + + - name: Test lookup with missing file + set_fact: + sops_file_does_not_exists: "{{ lookup('community.general.sops', 'file-does-not-exists.sops.yml') }}" + ignore_errors: yes + register: sops_lookup_missing_file + + - assert: + that: + - "sops_lookup_missing_file is failed" + - "'could not locate file in lookup: file-does-not-exists.sops.yml' in sops_lookup_missing_file.msg" + + - name: Test lookup of non-sops file + set_fact: + sops_wrong_file: "{{ lookup('community.general.sops', 'wrong.yaml') }}" + ignore_errors: yes + register: sops_lookup_wrong_file + + - assert: + that: + - "sops_lookup_wrong_file is failed" + - "'sops metadata not found' in sops_lookup_wrong_file.msg" + + - name: Test simple lookup + set_fact: + sops_success: "{{ lookup('community.general.sops', 'simple.sops.yaml') }}" + ignore_errors: yes + register: sops_lookup_simple + + - assert: + that: + - "sops_lookup_simple is success" + - "sops_success == 'foo: bar'" diff --git a/tests/integration/targets/setup_sops/meta/main.yml b/tests/integration/targets/setup_sops/meta/main.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/tests/integration/targets/setup_sops/meta/main.yml @@ -0,0 +1 @@ +--- diff --git a/tests/integration/targets/setup_sops/tasks/main.yml b/tests/integration/targets/setup_sops/tasks/main.yml new file mode 100644 index 0000000000..e38b11b492 --- /dev/null +++ b/tests/integration/targets/setup_sops/tasks/main.yml @@ -0,0 +1,7 @@ +--- + +- set_fact: + sops_installed: no + +- include: ubuntu.yml + when: ansible_distribution == 'Ubuntu' diff --git a/tests/integration/targets/setup_sops/tasks/ubuntu.yml b/tests/integration/targets/setup_sops/tasks/ubuntu.yml new file mode 100644 index 0000000000..1919465145 --- /dev/null +++ b/tests/integration/targets/setup_sops/tasks/ubuntu.yml @@ -0,0 +1,20 @@ +--- + +- name: install sops release + apt: + deb: https://github.com/mozilla/sops/releases/download/v3.5.0/sops_3.5.0_amd64.deb + +- name: install gnupg2 + apt: + package: gnupg2 + +- name: download sops test GPG key + get_url: + url: https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc + dest: /tmp/sops_functional_tests_key.asc + +- name: import sops test GPG key + command: gpg --import /tmp/sops_functional_tests_key.asc + +- set_fact: + sops_installed: yes