ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

Cyptography pr 20566 rebase (#25560)

Browse source 
Make pyca/cryptography the preferred backend for cryptographic needs (mainly vault) falling back to pycrypto

pyca/cryptography is already implicitly a dependency in many cases
through paramiko (2.0+) as well as the new openssl_publickey module,
which requires pyOpenSSL 16.0+. Additionally, pyca/cryptography is
an optional dep for better performance with vault already.

This commit leverages cryptography's padding, constant time comparisons,
and CBC/CTR modes to reduce the amount of code ansible needs to
maintain.

* Handle wrong password given for VaultAES format

* Do not display deprecation warning for cryptography on python-2.6

* Namespace all of the pycrypto imports and always import them

  Makes unittests better and the code less likely to get stupid mistakes
  (like using HMAC from cryptogrpahy when the one from pycrypto is needed)

* Add back in atfork since we need pycrypto to reinitialize its RNG just in case we're being used with old paramiko

* contrib/inventory/gce: Remove spurious require on pycrypto

(cherry picked from commit 9e16b9db275263b3ea8d1b124966fdebfc9ab271)

* Add cryptography to ec2_win_password module requirements
  * Fix python3 bug which would pass text strings to a function which
    requires byte strings.

* Attempt to add pycrypto version to setup deps

* Change hacking README for dual pycrypto/cryptography

* update dependencies for various CI scripts

* additional CI dockerfile/script updates

* add paramiko to the windows and sanity requirement set

  This is needed because ansible lists it as a requirement. Previously
  the missing dep wasn't enforced, but cryptography imports pkg_resources
  so you can't ignore a requirement any more

* Add integration test cases for old vault and for wrong passwords

* helper script for manual testing of pycrypto/cryptography

* Skip the pycrypto tests so that users without it installed can still run the unittests

* Run unittests for vault with both cryptography and pycrypto backend
This commit is contained in:
Toshio Kuratomi 2017-06-27 06:00:15 -07:00 committed by GitHub
parent e7e091d154
commit e238ae999b
25 changed files with 456 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -68,6 +68,8 @@ Ansible Changes By Release
* Experimentally added pmrun become method.
* Enable the docker connection plugin to use su as a become method
* Add an encoding parameter for the replace module so that it can operate on non-utf-8 files
* By default, Ansible now uses the cryptography module to implement vault
instead of the older pycrypto module.
#### New Callbacks:
- profile_roles
@ -92,6 +94,7 @@ Ansible Changes By Release
- The docker_container module has gained a new option, working_dir which allows
specifying the working directory for the command being run in the image.
- The ec2_win_password module now requires the cryptography python module be installed to run
### New Modules

1
contrib/inventory/gce.py
View file

@ -74,7 +74,6 @@ Contributors: Matt Hite <mhite@hotmail.com>, Tom Melendez <supertom@google.com>
Version: 0.0.3
'''
__requires__ = ['pycrypto>=2.6']
try:
import pkg_resources
except ImportError:

2
hacking/README.md
View file

@ -17,7 +17,7 @@ and do not wish to install them from your operating system package manager, you
can install them from pip
$ easy_install pip # if pip is not already available
$ pip install pyyaml jinja2 nose pytest passlib pycrypto
$ pip install -r requirements.txt
From there, follow ansible instructions on docs.ansible.com as normal.

14
lib/ansible/executor/process/worker.py
View file

@ -26,13 +26,15 @@ import traceback
from jinja2.exceptions import TemplateNotFound
# TODO: not needed if we use the cryptography library with its default RNG
# engine
HAS_ATFORK = True
HAS_PYCRYPTO_ATFORK = False
try:
from Crypto.Random import atfork
except ImportError:
HAS_ATFORK = False
HAS_PYCRYPTO_ATFORK = True
except:
# We only need to call atfork if pycrypto is used because it will need to
# reinitialize its RNG. Since old paramiko could be using pycrypto, we
# need to take charge of calling it.
pass
from ansible.errors import AnsibleConnectionFailure
from ansible.executor.task_executor import TaskExecutor
@ -99,7 +101,7 @@ class WorkerProcess(multiprocessing.Process):
# pr = cProfile.Profile()
# pr.enable()
if HAS_ATFORK:
if HAS_PYCRYPTO_ATFORK:
atfork()
try:

29
lib/ansible/modules/cloud/amazon/ec2_win_password.py
View file

@ -60,6 +60,13 @@ options:
extends_documentation_fragment:
- aws
- ec2
requirements:
- cryptography
notes:
- As of Ansible 2.4, this module requires the python cryptography module rather than the
older pycrypto module.
'''
EXAMPLES = '''
@ -95,9 +102,11 @@ tasks:
'''
from base64 import b64decode
from Crypto.Cipher import PKCS1_v1_5
from Crypto.PublicKey import RSA
from os.path import expanduser
import datetime
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15
from cryptography.hazmat.primitives.serialization import load_pem_private_key
try:
import boto.ec2
@ -105,6 +114,9 @@ try:
except ImportError:
HAS_BOTO = False
BACKEND = default_backend()
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(dict(
@ -122,7 +134,7 @@ def main():
instance_id = module.params.get('instance_id')
key_file = module.params.get('key_file')
key_passphrase = module.params.get('key_passphrase')
b_key_passphrase = to_bytes(module.params.get('key_passphrase'), errors='surrogate_or_strict')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
@ -147,21 +159,18 @@ def main():
module.fail_json(msg = "wait for password timeout after %d seconds" % wait_timeout)
try:
f = open(key_file, 'r')
f = open(key_file, 'rb')
except IOError as e:
module.fail_json(msg = "I/O error (%d) opening key file: %s" % (e.errno, e.strerror))
else:
try:
with f:
key = RSA.importKey(f.read(), key_passphrase)
except (ValueError, IndexError, TypeError) as e:
key = load_pem_private_key(f.read(), b_key_passphrase, BACKEND)
except (ValueError, TypeError) as e:
module.fail_json(msg = "unable to parse key file")
cipher = PKCS1_v1_5.new(key)
sentinel = 'password decryption failed!!!'
try:
decrypted = cipher.decrypt(decoded, sentinel)
decrypted = key.decrypt(decoded, PKCS1v15())
except ValueError as e:
decrypted = None

372
lib/ansible/parsing/vault/__init__.py
View file

@ -1,6 +1,6 @@
# (c) 2014, James Tanner <tanner.jc@gmail.com>
# (c) 2016, Adrian Likins <alikins@redhat.com>
# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
# (c) 2016 Toshio Kuratomi <tkuratomi@ansible.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -20,47 +20,56 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import random
import shlex
import shutil
import sys
import tempfile
import random
from io import BytesIO
from subprocess import call
from hashlib import sha256
import warnings
from binascii import hexlify
from binascii import unhexlify
from hashlib import md5
from hashlib import sha256
from io import BytesIO
from subprocess import call
# Note: Only used for loading obsolete VaultAES files. All files are written
# using the newer VaultAES256 which does not require md5
HAS_CRYPTOGRAPHY = False
HAS_PYCRYPTO = False
HAS_SOME_PYCRYPTO = False
CRYPTOGRAPHY_BACKEND = None
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, padding
from cryptography.hazmat.primitives.hmac import HMAC
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from cryptography.hazmat.primitives.ciphers import (
Cipher as C_Cipher, algorithms, modes
)
CRYPTOGRAPHY_BACKEND = default_backend()
HAS_CRYPTOGRAPHY = True
except ImportError:
pass
try:
from Crypto.Hash import SHA256, HMAC
HAS_HASH = True
except ImportError:
HAS_HASH = False
from Crypto.Cipher import AES as AES_pycrypto
HAS_SOME_PYCRYPTO = True
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
from Crypto.Util import Counter
HAS_COUNTER = True
except ImportError:
HAS_COUNTER = False
# Note: Only used for loading obsolete VaultAES files. All files are written
# using the newer VaultAES256 which does not require md5
from Crypto.Hash import SHA256 as SHA256_pycrypto
from Crypto.Hash import HMAC as HMAC_pycrypto
# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
try:
from Crypto.Protocol.KDF import PBKDF2
HAS_PBKDF2 = True
except ImportError:
HAS_PBKDF2 = False
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
from Crypto.Util import Counter as Counter_pycrypto
# AES IMPORTS
try:
from Crypto.Cipher import AES as AES
HAS_AES = True
# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
from Crypto.Protocol.KDF import PBKDF2 as PBKDF2_pycrypto
HAS_PYCRYPTO = True
except ImportError:
HAS_AES = False
pass
from ansible.errors import AnsibleError
from ansible.module_utils.six import PY3, binary_type
@ -73,25 +82,6 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
# OpenSSL pbkdf2_hmac
HAS_PBKDF2HMAC = False
try:
from cryptography.hazmat.primitives.hashes import SHA256 as c_SHA256
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from cryptography.hazmat.backends import default_backend
HAS_PBKDF2HMAC = True
except ImportError:
pass
except Exception as e:
display.vvvv("Optional dependency 'cryptography' raised an exception, falling back to 'Crypto'.")
import traceback
display.vvvv("Traceback from import of cryptography was {0}".format(traceback.format_exc()))
HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC
CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform." \
" You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"
b_HEADER = b'$ANSIBLE_VAULT'
CIPHER_WHITELIST = frozenset((u'AES', u'AES256'))
@ -99,11 +89,10 @@ CIPHER_WRITE_WHITELIST = frozenset((u'AES256',))
# See also CIPHER_MAPPING at the bottom of the file which maps cipher strings
# (used in VaultFile header) to a cipher class
def check_prereqs():
if not HAS_AES or not HAS_COUNTER or not HAS_ANY_PBKDF2HMAC or not HAS_HASH:
raise AnsibleError(CRYPTO_UPGRADE)
NEED_CRYPTO_LIBRARY = "ansible-vault requires either the cryptography library (preferred) or"
if HAS_SOME_PYCRYPTO:
NEED_CRYPTO_LIBRARY += " a newer version of"
NEED_CRYPTO_LIBRARY += " pycrypto in order to function."
class AnsibleVaultError(AnsibleError):
@ -411,7 +400,6 @@ class VaultEditor:
return real_path
def encrypt_bytes(self, b_plaintext):
check_prereqs()
b_ciphertext = self.vault.encrypt(b_plaintext)
@ -419,8 +407,6 @@ class VaultEditor:
def encrypt_file(self, filename, output_file=None):
check_prereqs()
# A file to be encrypted into a vaultfile could be any encoding
# so treat the contents as a byte string.
@ -433,8 +419,6 @@ class VaultEditor:
def decrypt_file(self, filename, output_file=None):
check_prereqs()
# follow the symlink
filename = self._real_path(filename)
@ -449,8 +433,6 @@ class VaultEditor:
def create_file(self, filename):
""" create a new encrypted file """
check_prereqs()
# FIXME: If we can raise an error here, we can probably just make it
# behave like edit instead.
if os.path.isfile(filename):
@ -460,8 +442,6 @@ class VaultEditor:
def edit_file(self, filename):
check_prereqs()
# follow the symlink
filename = self._real_path(filename)
@ -480,7 +460,6 @@ class VaultEditor:
def plaintext(self, filename):
check_prereqs()
ciphertext = self.read_data(filename)
try:
@ -492,8 +471,6 @@ class VaultEditor:
def rekey_file(self, filename, b_new_password):
check_prereqs()
# follow the symlink
filename = self._real_path(filename)
@ -609,10 +586,11 @@ class VaultAES:
# Note: strings in this class should be byte strings by default.
def __init__(self):
if not HAS_AES:
raise AnsibleError(CRYPTO_UPGRADE)
if not HAS_CRYPTOGRAPHY and not HAS_PYCRYPTO:
raise AnsibleError(NEED_CRYPTO_LIBRARY)
def _aes_derive_key_and_iv(self, b_password, b_salt, key_length, iv_length):
@staticmethod
def _aes_derive_key_and_iv(b_password, b_salt, key_length, iv_length):
""" Create a key and an initialization vector """
@ -627,37 +605,49 @@ class VaultAES:
return b_key, b_iv
def encrypt(self, b_plaintext, b_password, key_length=32):
@staticmethod
def encrypt(b_plaintext, b_password, key_length=32):
""" Read plaintext data from in_file and write encrypted to out_file """
raise AnsibleError("Encryption disabled for deprecated VaultAES class")
def decrypt(self, b_vaulttext, b_password, key_length=32):
@classmethod
def _decrypt_cryptography(cls, b_salt, b_ciphertext, b_password, key_length):
bs = algorithms.AES.block_size // 8
b_key, b_iv = cls._aes_derive_key_and_iv(b_password, b_salt, key_length, bs)
cipher = C_Cipher(algorithms.AES(b_key), modes.CBC(b_iv), CRYPTOGRAPHY_BACKEND).decryptor()
unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder()
""" Decrypt the given data and return it
:arg b_data: A byte string containing the encrypted data
:arg b_password: A byte string containing the encryption password
:arg key_length: Length of the key
:returns: A byte string containing the decrypted data
"""
try:
b_plaintext = unpadder.update(
cipher.update(b_ciphertext) + cipher.finalize()
) + unpadder.finalize()
except ValueError:
# In VaultAES, ValueError: invalid padding bytes can mean bad
# password was given
raise AnsibleError("Decryption failed")
display.deprecated(u'The VaultAES format is insecure and has been '
'deprecated since Ansible-1.5. Use vault rekey FILENAME to '
'switch to the newer VaultAES256 format', version='2.3')
# http://stackoverflow.com/a/14989032
# split out sha and verify decryption
b_split_data = b_plaintext.split(b"\n", 1)
b_this_sha = b_split_data[0]
b_plaintext = b_split_data[1]
b_test_sha = to_bytes(sha256(b_plaintext).hexdigest())
b_ciphertext = unhexlify(b_vaulttext)
if b_this_sha != b_test_sha:
raise AnsibleError("Decryption failed")
return b_plaintext
@classmethod
def _decrypt_pycrypto(cls, b_salt, b_ciphertext, b_password, key_length):
in_file = BytesIO(b_ciphertext)
in_file.seek(0)
out_file = BytesIO()
bs = AES.block_size
b_tmpsalt = in_file.read(bs)
b_salt = b_tmpsalt[len(b'Salted__'):]
b_key, b_iv = self._aes_derive_key_and_iv(b_password, b_salt, key_length, bs)
cipher = AES.new(b_key, AES.MODE_CBC, b_iv)
bs = AES_pycrypto.block_size
b_key, b_iv = cls._aes_derive_key_and_iv(b_password, b_salt, key_length, bs)
cipher = AES_pycrypto.new(b_key, AES_pycrypto.MODE_CBC, b_iv)
b_next_chunk = b''
finished = False
@ -691,6 +681,34 @@ class VaultAES:
return b_plaintext
@classmethod
def decrypt(cls, b_vaulttext, b_password, key_length=32):
""" Decrypt the given data and return it
:arg b_data: A byte string containing the encrypted data
:arg b_password: A byte string containing the encryption password
:arg key_length: Length of the key
:returns: A byte string containing the decrypted data
"""
display.deprecated(u'The VaultAES format is insecure and has been '
'deprecated since Ansible-1.5. Use vault rekey FILENAME to '
'switch to the newer VaultAES256 format', version='2.3')
# http://stackoverflow.com/a/14989032
b_vaultdata = unhexlify(b_vaulttext)
b_salt = b_vaultdata[len(b'Salted__'):16]
b_ciphertext = b_vaultdata[16:]
if HAS_CRYPTOGRAPHY:
b_plaintext = cls._decrypt_cryptography(b_salt, b_ciphertext, b_password, key_length)
elif HAS_PYCRYPTO:
b_plaintext = cls._decrypt_pycrypto(b_salt, b_ciphertext, b_password, key_length)
else:
raise AnsibleError(NEED_CRYPTO_LIBRARY + ' (Late detection)')
return b_plaintext
class VaultAES256:
@ -704,53 +722,79 @@ class VaultAES256:
# Note: strings in this class should be byte strings by default.
def __init__(self):
check_prereqs()
if not HAS_CRYPTOGRAPHY and not HAS_PYCRYPTO:
raise AnsibleError(NEED_CRYPTO_LIBRARY)
@staticmethod
def _create_key(b_password, b_salt, keylength, ivlength):
hash_function = SHA256
def _create_key_cryptography(b_password, b_salt, key_length, iv_length):
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=2 * key_length + iv_length,
salt=b_salt,
iterations=10000,
backend=CRYPTOGRAPHY_BACKEND)
b_derivedkey = kdf.derive(b_password)
return b_derivedkey
@staticmethod
def _pbkdf2_prf(p, s):
hash_function = SHA256_pycrypto
return HMAC_pycrypto.new(p, s, hash_function).digest()
@classmethod
def _create_key_pycrypto(cls, b_password, b_salt, key_length, iv_length):
# make two keys and one iv
def pbkdf2_prf(p, s):
return HMAC.new(p, s, hash_function).digest()
b_derivedkey = PBKDF2(b_password, b_salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
b_derivedkey = PBKDF2_pycrypto(b_password, b_salt, dkLen=(2 * key_length) + iv_length,
count=10000, prf=cls._pbkdf2_prf)
return b_derivedkey
@classmethod
def _gen_key_initctr(cls, b_password, b_salt):
# 16 for AES 128, 32 for AES256
keylength = 32
key_length = 32
# match the size used for counter.new to avoid extra work
ivlength = 16
if HAS_CRYPTOGRAPHY:
# AES is a 128-bit block cipher, so IVs and counter nonces are 16 bytes
iv_length = algorithms.AES.block_size // 8
if HAS_PBKDF2HMAC:
backend = default_backend()
kdf = PBKDF2HMAC(
algorithm=c_SHA256(),
length=2 * keylength + ivlength,
salt=b_salt,
iterations=10000,
backend=backend)
b_derivedkey = kdf.derive(b_password)
b_derivedkey = cls._create_key_cryptography(b_password, b_salt, key_length, iv_length)
b_iv = b_derivedkey[(key_length * 2):(key_length * 2) + iv_length]
elif HAS_PYCRYPTO:
# match the size used for counter.new to avoid extra work
iv_length = 16
b_derivedkey = cls._create_key_pycrypto(b_password, b_salt, key_length, iv_length)
b_iv = hexlify(b_derivedkey[(key_length * 2):(key_length * 2) + iv_length])
else:
b_derivedkey = cls._create_key(b_password, b_salt, keylength, ivlength)
raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in initctr)')
b_key1 = b_derivedkey[:keylength]
b_key2 = b_derivedkey[keylength:(keylength * 2)]
b_iv = b_derivedkey[(keylength * 2):(keylength * 2) + ivlength]
b_key1 = b_derivedkey[:key_length]
b_key2 = b_derivedkey[key_length:(key_length * 2)]
return b_key1, b_key2, hexlify(b_iv)
return b_key1, b_key2, b_iv
def encrypt(self, b_plaintext, b_password):
b_salt = os.urandom(32)
b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt)
@staticmethod
def _encrypt_cryptography(b_plaintext, b_salt, b_key1, b_key2, b_iv):
cipher = C_Cipher(algorithms.AES(b_key1), modes.CTR(b_iv), CRYPTOGRAPHY_BACKEND)
encryptor = cipher.encryptor()
padder = padding.PKCS7(algorithms.AES.block_size).padder()
b_ciphertext = encryptor.update(padder.update(b_plaintext) + padder.finalize())
b_ciphertext += encryptor.finalize()
# COMBINE SALT, DIGEST AND DATA
hmac = HMAC(b_key2, hashes.SHA256(), CRYPTOGRAPHY_BACKEND)
hmac.update(b_ciphertext)
b_hmac = hmac.finalize()
return hexlify(b_hmac), hexlify(b_ciphertext)
@staticmethod
def _encrypt_pycrypto(b_plaintext, b_salt, b_key1, b_key2, b_iv):
# PKCS#7 PAD DATA http://tools.ietf.org/html/rfc5652#section-6.3
bs = AES.block_size
bs = AES_pycrypto.block_size
padding_length = (bs - len(b_plaintext) % bs) or bs
b_plaintext += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict')
@ -758,50 +802,58 @@ class VaultAES256:
# 1) nbits (integer) - Length of the counter, in bits.
# 2) initial_value (integer) - initial value of the counter. "iv" from _gen_key_initctr
ctr = Counter.new(128, initial_value=int(b_iv, 16))
ctr = Counter_pycrypto.new(128, initial_value=int(b_iv, 16))
# AES.new PARAMETERS
# 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from _gen_key_initctr
# 2) MODE_CTR, is the recommended mode
# 3) counter=<CounterObject>
cipher = AES.new(b_key1, AES.MODE_CTR, counter=ctr)
cipher = AES_pycrypto.new(b_key1, AES_pycrypto.MODE_CTR, counter=ctr)
# ENCRYPT PADDED DATA
b_ciphertext = cipher.encrypt(b_plaintext)
# COMBINE SALT, DIGEST AND DATA
hmac = HMAC.new(b_key2, b_ciphertext, SHA256)
b_vaulttext = b'\n'.join([hexlify(b_salt), to_bytes(hmac.hexdigest()), hexlify(b_ciphertext)])
hmac = HMAC_pycrypto.new(b_key2, b_ciphertext, SHA256_pycrypto)
return to_bytes(hmac.hexdigest(), errors='surrogate_or_strict'), hexlify(b_ciphertext)
@classmethod
def encrypt(cls, b_plaintext, b_password):
b_salt = os.urandom(32)
b_key1, b_key2, b_iv = cls._gen_key_initctr(b_password, b_salt)
if HAS_CRYPTOGRAPHY:
b_hmac, b_ciphertext = cls._encrypt_cryptography(b_plaintext, b_salt, b_key1, b_key2, b_iv)
elif HAS_PYCRYPTO:
b_hmac, b_ciphertext = cls._encrypt_pycrypto(b_plaintext, b_salt, b_key1, b_key2, b_iv)
else:
raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in encrypt)')
b_vaulttext = b'\n'.join([hexlify(b_salt), b_hmac, b_ciphertext])
# Unnecessary but getting rid of it is a backwards incompatible vault
# format change
b_vaulttext = hexlify(b_vaulttext)
return b_vaulttext
def decrypt(self, b_vaulttext, b_password):
# SPLIT SALT, DIGEST, AND DATA
b_vaulttext = unhexlify(b_vaulttext)
b_salt, b_cryptedHmac, b_ciphertext = b_vaulttext.split(b"\n", 2)
b_salt = unhexlify(b_salt)
b_ciphertext = unhexlify(b_ciphertext)
b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt)
@staticmethod
def _decrypt_cryptography(b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv):
# EXIT EARLY IF DIGEST DOESN'T MATCH
hmacDecrypt = HMAC.new(b_key2, b_ciphertext, SHA256)
if not self._is_equal(b_cryptedHmac, to_bytes(hmacDecrypt.hexdigest())):
hmac = HMAC(b_key2, hashes.SHA256(), CRYPTOGRAPHY_BACKEND)
hmac.update(b_ciphertext)
try:
hmac.verify(unhexlify(b_crypted_hmac))
except InvalidSignature:
return None
# SET THE COUNTER AND THE CIPHER
ctr = Counter.new(128, initial_value=int(b_iv, 16))
cipher = AES.new(b_key1, AES.MODE_CTR, counter=ctr)
# DECRYPT PADDED DATA
b_plaintext = cipher.decrypt(b_ciphertext)
cipher = C_Cipher(algorithms.AES(b_key1), modes.CTR(b_iv), CRYPTOGRAPHY_BACKEND)
decryptor = cipher.decryptor()
unpadder = padding.PKCS7(128).unpadder()
b_plaintext = unpadder.update(
decryptor.update(b_ciphertext) + decryptor.finalize()
) + unpadder.finalize()
# UNPAD DATA
if PY3:
padding_length = b_plaintext[-1]
else:
padding_length = ord(b_plaintext[-1])
b_plaintext = b_plaintext[:-padding_length]
return b_plaintext
@staticmethod
@ -828,6 +880,46 @@ class VaultAES256:
result |= ord(b_x) ^ ord(b_y)
return result == 0
@classmethod
def _decrypt_pycrypto(cls, b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv):
# EXIT EARLY IF DIGEST DOESN'T MATCH
hmac_decrypt = HMAC_pycrypto.new(b_key2, b_ciphertext, SHA256_pycrypto)
if not cls._is_equal(b_crypted_hmac, to_bytes(hmac_decrypt.hexdigest())):
return None
# SET THE COUNTER AND THE CIPHER
ctr = Counter_pycrypto.new(128, initial_value=int(b_iv, 16))
cipher = AES_pycrypto.new(b_key1, AES_pycrypto.MODE_CTR, counter=ctr)
# DECRYPT PADDED DATA
b_plaintext = cipher.decrypt(b_ciphertext)
# UNPAD DATA
if PY3:
padding_length = b_plaintext[-1]
else:
padding_length = ord(b_plaintext[-1])
b_plaintext = b_plaintext[:-padding_length]
return b_plaintext
@classmethod
def decrypt(cls, b_vaulttext, b_password):
# SPLIT SALT, DIGEST, AND DATA
b_vaulttext = unhexlify(b_vaulttext)
b_salt, b_crypted_hmac, b_ciphertext = b_vaulttext.split(b"\n", 2)
b_salt = unhexlify(b_salt)
b_ciphertext = unhexlify(b_ciphertext)
b_key1, b_key2, b_iv = cls._gen_key_initctr(b_password, b_salt)
if HAS_CRYPTOGRAPHY:
b_plaintext = cls._decrypt_cryptography(b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv)
elif HAS_PYCRYPTO:
b_plaintext = cls._decrypt_pycrypto(b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv)
else:
raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in decrypt)')
return b_plaintext
# Keys could be made bytes later if the code that gets the data is more
# naturally byte-oriented

2
requirements.txt
View file

@ -6,5 +6,5 @@
jinja2
PyYAML
paramiko
pycrypto >= 2.6
cryptography
setuptools

4
setup.py
View file

@ -25,6 +25,10 @@ with open('requirements.txt') as requirements_file:
# knows about
crypto_backend = os.environ.get('ANSIBLE_CRYPTO_BACKEND', None)
if crypto_backend:
if crypto_backend.strip() == 'pycrypto':
# Attempt to set version requirements
crypto_backend = 'pycrypto >= 2.6'
install_requirements = [r for r in install_requirements if not (r.lower().startswith('pycrypto') or r.lower().startswith('cryptography'))]
install_requirements.append(crypto_backend)

4
test/integration/targets/vault/format_1_0_AES.yml Normal file
View file

@ -0,0 +1,4 @@
$ANSIBLE_VAULT;1.0;AES
53616c7465645f5fd0026926a2d415a28a2622116273fbc90e377225c12a347e1daf4456d36a77f9
9ad98d59f61d06a4b66718d855f16fb7bdfe54d1ec8aeaa4d06c2dc1fa630ae1846a029877f0eeb1
83c62ffb04c2512995e815de4b4d29ed

4
test/integration/targets/vault/format_1_1_AES.yml Normal file
View file

@ -0,0 +1,4 @@
$ANSIBLE_VAULT;1.1;AES
53616c7465645f5fc107ce1ef4d7b455e038a13b053225776458052f8f8f332d554809d3f150bfa3
fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
786a5a15efeb787e1958cbdd480d076c

6
test/integration/targets/vault/format_1_1_AES256.yml Normal file
View file

@ -0,0 +1,6 @@
$ANSIBLE_VAULT;1.1;AES256
33613463343938323434396164663236376438313435633837336438366530666431643031333734
6463646538393331333239393363333830613039376562360a396635393636636539346332336364
35303039353164386461326439346165656463383137663932323930666632326263636266656461
3232663537653637640a643166666232633936636664376435316664656631633166323237356163
6138

27
test/integration/targets/vault/runme.sh
View file

@ -11,6 +11,33 @@ echo "This is a test file" > "${TEST_FILE}"
TEST_FILE_OUTPUT="${MYTMPDIR}/test_file_output"
# old format
ansible-vault view "$@" --vault-password-file vault-password-ansible format_1_0_AES.yml
ansible-vault view "$@" --vault-password-file vault-password-ansible format_1_1_AES.yml
# old format, wrong password
echo "The wrong password tests are expected to return 1"
ansible-vault view "$@" --vault-password-file vault-password-wrong format_1_0_AES.yml && :
WRONG_RC=$?
echo "rc was $WRONG_RC (1 is expected)"
[ $WRONG_RC -eq 1 ]
ansible-vault view "$@" --vault-password-file vault-password-wrong format_1_1_AES.yml && :
WRONG_RC=$?
echo "rc was $WRONG_RC (1 is expected)"
[ $WRONG_RC -eq 1 ]
ansible-vault view "$@" --vault-password-file vault-password-wrong format_1_1_AES256.yml && :
WRONG_RC=$?
echo "rc was $WRONG_RC (1 is expected)"
[ $WRONG_RC -eq 1 ]
set -eux
# new format, view
ansible-vault view "$@" --vault-password-file vault-password format_1_1_AES256.yml
# encrypt it
ansible-vault encrypt "$@" --vault-password-file vault-password "${TEST_FILE}"

27
test/integration/targets/vault/runme_change_pip_installed.sh Executable file
View file

@ -0,0 +1,27 @@
#!/bin/bash
# start by removing pycrypto and cryptography
pip uninstall -y cryptography
pip uninstall -y pycrypto
./runme.sh
# now just pycrypto
pip install --user pycrypto
./runme.sh
# now just cryptography
pip uninstall -y pycrypto
pip install --user cryptography
./runme.sh
# now both
pip install --user pycrypto
./runme.sh

1
test/integration/targets/vault/vault-password-ansible Normal file
View file

@ -0,0 +1 @@
ansible

1
test/integration/targets/vault/vault-password-wrong Normal file
View file

@ -0,0 +1 @@
hunter42

1
test/runner/requirements/constraints.txt
View file

@ -3,3 +3,4 @@ pywinrm >= 0.2.1 # 0.1.1 required, but 0.2.1 provides better performance
pylint >= 1.5.3, < 1.7.0 # 1.4.1 adds JSON output, but 1.5.3 fixes bugs related to JSON output
sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7 or later
isort < 4.2.8 # 4.2.8 changes import sort order requirements which breaks previously passing pylint tests
pycrypto >= 2.6 # Need features found in 2.6 and greater

1
test/runner/requirements/integration.txt
View file

@ -1,3 +1,4 @@
cryptography
jinja2
jmespath
junit-xml

2
test/runner/requirements/network-integration.txt
View file

@ -1,5 +1,5 @@
cryptography
jinja2
junit-xml
paramiko
pycrypto
pyyaml

2
test/runner/requirements/sanity.txt
View file

@ -1,6 +1,8 @@
cryptography
jinja2
mock
pep8
paramiko
pylint
pytest
rstcheck

3
test/runner/requirements/units.txt
View file

@ -1,11 +1,12 @@
boto
boto3
placebo
cryptography
pycrypto
jinja2
mock
nose
passlib
pycrypto
pytest
pytest-mock
pytest-xdist

2
test/runner/requirements/windows-integration.txt
View file

@ -1,4 +1,6 @@
cryptography
jinja2
junit-xml
paramiko
pywinrm
pyyaml

136
test/units/parsing/vault/test_vault.py
View file

@ -26,7 +26,7 @@ import io
import os
from binascii import hexlify
from nose.plugins.skip import SkipTest
import pytest
from ansible.compat.tests import unittest
@ -37,28 +37,6 @@ from ansible.parsing.vault import VaultLib
from ansible.parsing import vault
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
from Crypto.Util import Counter
HAS_COUNTER = True
except ImportError:
HAS_COUNTER = False
# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
try:
from Crypto.Protocol.KDF import PBKDF2
HAS_PBKDF2 = True
except ImportError:
HAS_PBKDF2 = False
# AES IMPORTS
try:
from Crypto.Cipher import AES as AES
HAS_AES = True
except ImportError:
HAS_AES = False
class TestVaultIsEncrypted(unittest.TestCase):
def test_bytes_not_encrypted(self):
b_data = b"foobar"
@ -151,6 +129,8 @@ class TestVaultIsEncryptedFile(unittest.TestCase):
self.assertTrue(vault.is_encrypted_file(b_data_fo, start_pos=4, count=vault_length))
@pytest.mark.skipif(not vault.HAS_CRYPTOGRAPHY,
reason="Skipping cryptography tests because cryptography is not installed")
class TestVaultCipherAes256(unittest.TestCase):
def setUp(self):
self.vault_cipher = vault.VaultAES256()
@ -159,26 +139,71 @@ class TestVaultCipherAes256(unittest.TestCase):
self.assertIsInstance(self.vault_cipher, vault.VaultAES256)
# TODO: tag these as slow tests
def test_create_key(self):
def test_create_key_cryptography(self):
b_password = b'hunter42'
b_salt = os.urandom(32)
b_key = self.vault_cipher._create_key(b_password, b_salt, keylength=32, ivlength=16)
self.assertIsInstance(b_key, six.binary_type)
b_key_cryptography = self.vault_cipher._create_key_cryptography(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_cryptography, six.binary_type)
def test_create_key_known(self):
@pytest.mark.skipif(not vault.HAS_PYCRYPTO, reason='Not testing pycrypto key as pycrypto is not installed')
def test_create_key_pycrypto(self):
b_password = b'hunter42'
b_salt = os.urandom(32)
b_key_pycrypto = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_pycrypto, six.binary_type)
@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
reason='Not comparing cryptography key to pycrypto key as pycrypto is not installed')
def test_compare_new_keys(self):
b_password = b'hunter42'
b_salt = os.urandom(32)
b_key_cryptography = self.vault_cipher._create_key_cryptography(b_password, b_salt, key_length=32, iv_length=16)
b_key_pycrypto = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
self.assertEqual(b_key_cryptography, b_key_pycrypto)
def test_create_key_known_cryptography(self):
b_password = b'hunter42'
# A fixed salt
b_salt = b'q' * 32 # q is the most random letter.
b_key = self.vault_cipher._create_key(b_password, b_salt, keylength=32, ivlength=16)
self.assertIsInstance(b_key, six.binary_type)
b_key_1 = self.vault_cipher._create_key_cryptography(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_1, six.binary_type)
# verify we get the same answer
# we could potentially run a few iterations of this and time it to see if it's roughly constant time
# and or that it exceeds some minimal time, but that would likely cause unreliable fails, esp in CI
b_key_2 = self.vault_cipher._create_key(b_password, b_salt, keylength=32, ivlength=16)
self.assertIsInstance(b_key, six.binary_type)
self.assertEqual(b_key, b_key_2)
b_key_2 = self.vault_cipher._create_key_cryptography(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_2, six.binary_type)
self.assertEqual(b_key_1, b_key_2)
# And again with pycrypto
b_key_3 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_3, six.binary_type)
# verify we get the same answer
# we could potentially run a few iterations of this and time it to see if it's roughly constant time
# and or that it exceeds some minimal time, but that would likely cause unreliable fails, esp in CI
b_key_4 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_4, six.binary_type)
self.assertEqual(b_key_3, b_key_4)
self.assertEqual(b_key_1, b_key_4)
def test_create_key_known_pycrypto(self):
b_password = b'hunter42'
# A fixed salt
b_salt = b'q' * 32 # q is the most random letter.
b_key_3 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_3, six.binary_type)
# verify we get the same answer
# we could potentially run a few iterations of this and time it to see if it's roughly constant time
# and or that it exceeds some minimal time, but that would likely cause unreliable fails, esp in CI
b_key_4 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
self.assertIsInstance(b_key_4, six.binary_type)
self.assertEqual(b_key_3, b_key_4)
def test_is_equal_is_equal(self):
self.assertTrue(self.vault_cipher._is_equal(b'abcdefghijklmnopqrstuvwxyz', b'abcdefghijklmnopqrstuvwxyz'))
@ -213,6 +238,21 @@ class TestVaultCipherAes256(unittest.TestCase):
self.assertRaises(TypeError, self.vault_cipher._is_equal, b"blue fish", 2)
@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
reason="Skipping Pycrypto tests because pycrypto is not installed")
class TestVaultCipherAes256PyCrypto(TestVaultCipherAes256):
def setUp(self):
self.has_cryptography = vault.HAS_CRYPTOGRAPHY
vault.HAS_CRYPTOGRAPHY = False
super(TestVaultCipherAes256PyCrypto, self).setUp()
def tearDown(self):
vault.HAS_CRYPTOGRAPHY = self.has_cryptography
super(TestVaultCipherAes256PyCrypto, self).tearDown()
@pytest.mark.skipif(not vault.HAS_CRYPTOGRAPHY,
reason="Skipping cryptography tests because cryptography is not installed")
class TestVaultLib(unittest.TestCase):
def setUp(self):
self.v = VaultLib('test-vault-password')
@ -266,8 +306,6 @@ class TestVaultLib(unittest.TestCase):
self.assertEqual(self.v.b_version, b"9.9", msg="version was not properly set")
def test_encrypt_decrypt_aes(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
self.v.cipher_name = u'AES'
self.v.b_password = b'ansible'
# AES encryption code has been removed, so this is old output for
@ -281,8 +319,6 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
self.assertEqual(b_plaintext, b"foobar", msg="decryption failed")
def test_encrypt_decrypt_aes256(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
self.v.cipher_name = u'AES256'
plaintext = u"foobar"
b_vaulttext = self.v.encrypt(plaintext)
@ -291,8 +327,6 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
self.assertEqual(b_plaintext, b"foobar", msg="decryption failed")
def test_encrypt_decrypt_aes256_existing_vault(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
self.v.cipher_name = u'AES256'
b_orig_plaintext = b"Setec Astronomy"
vaulttext = u'''$ANSIBLE_VAULT;1.1;AES256
@ -309,12 +343,10 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
b_plaintext = self.v.decrypt(b_vaulttext)
self.assertEqual(b_plaintext, b_orig_plaintext, msg="decryption failed")
# FIXME This test isn't working quite yet.
@pytest.mark.skip(reason='This test is not ready yet')
def test_encrypt_decrypt_aes256_bad_hmac(self):
# FIXME This test isn't working quite yet.
raise SkipTest
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
self.v.cipher_name = 'AES256'
# plaintext = "Setec Astronomy"
enc_data = '''$ANSIBLE_VAULT;1.1;AES256
@ -349,8 +381,6 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
self.v.decrypt(b_invalid_ciphertext)
def test_encrypt_encrypted(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
self.v.cipher_name = u'AES'
b_vaulttext = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
vaulttext = to_text(b_vaulttext, errors='strict')
@ -358,8 +388,6 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
self.assertRaises(errors.AnsibleError, self.v.encrypt, vaulttext)
def test_decrypt_decrypted(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
plaintext = u"ansible"
self.assertRaises(errors.AnsibleError, self.v.decrypt, plaintext)
@ -367,9 +395,19 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
self.assertRaises(errors.AnsibleError, self.v.decrypt, b_plaintext)
def test_cipher_not_set(self):
# not setting the cipher should default to AES256
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
plaintext = u"ansible"
self.v.encrypt(plaintext)
self.assertEquals(self.v.cipher_name, "AES256")
@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
reason="Skipping Pycrypto tests because pycrypto is not installed")
class TestVaultLibPyCrypto(TestVaultLib):
def setUp(self):
self.has_cryptography = vault.HAS_CRYPTOGRAPHY
vault.HAS_CRYPTOGRAPHY = False
super(TestVaultLibPyCrypto, self).setUp()
def tearDown(self):
vault.HAS_CRYPTOGRAPHY = self.has_cryptography
super(TestVaultLibPyCrypto, self).tearDown()

49
test/units/parsing/vault/test_vault_editor.py
View file

@ -22,7 +22,8 @@ __metaclass__ = type
import os
import tempfile
from nose.plugins.skip import SkipTest
import pytest
from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch
@ -32,27 +33,6 @@ from ansible.parsing import vault
from ansible.module_utils._text import to_bytes, to_text
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
from Crypto.Util import Counter
HAS_COUNTER = True
except ImportError:
HAS_COUNTER = False
# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
try:
from Crypto.Protocol.KDF import PBKDF2
HAS_PBKDF2 = True
except ImportError:
HAS_PBKDF2 = False
# AES IMPORTS
try:
from Crypto.Cipher import AES as AES
HAS_AES = True
except ImportError:
HAS_AES = False
v10_data = """$ANSIBLE_VAULT;1.0;AES
53616c7465645f5fd0026926a2d415a28a2622116273fbc90e377225c12a347e1daf4456d36a77f9
9ad98d59f61d06a4b66718d855f16fb7bdfe54d1ec8aeaa4d06c2dc1fa630ae1846a029877f0eeb1
@ -66,6 +46,8 @@ v11_data = """$ANSIBLE_VAULT;1.1;AES256
3739"""
@pytest.mark.skipif(not vault.HAS_CRYPTOGRAPHY,
reason="Skipping cryptography tests because cryptography is not installed")
class TestVaultEditor(unittest.TestCase):
def setUp(self):
@ -423,9 +405,6 @@ class TestVaultEditor(unittest.TestCase):
def test_decrypt_1_0(self):
# Skip testing decrypting 1.0 files if we don't have access to AES, KDF or Counter.
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
v10_file = tempfile.NamedTemporaryFile(delete=False)
with v10_file as f:
f.write(to_bytes(v10_data))
@ -451,9 +430,6 @@ class TestVaultEditor(unittest.TestCase):
assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
def test_decrypt_1_1(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
v11_file = tempfile.NamedTemporaryFile(delete=False)
with v11_file as f:
f.write(to_bytes(v11_data))
@ -478,10 +454,6 @@ class TestVaultEditor(unittest.TestCase):
assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
def test_rekey_migration(self):
# Skip testing rekeying files if we don't have access to AES, KDF or Counter.
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
v10_file = tempfile.NamedTemporaryFile(delete=False)
with v10_file as f:
f.write(to_bytes(v10_data))
@ -542,3 +514,16 @@ class TestVaultEditor(unittest.TestCase):
res = ve._real_path(file_link_path)
self.assertEqual(res, file_path)
@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
reason="Skipping pycrypto tests because pycrypto is not installed")
class TestVaultEditorPyCrypto(unittest.TestCase):
def setUp(self):
self.has_cryptography = vault.HAS_CRYPTOGRAPHY
vault.HAS_CRYPTOGRAPHY = False
super(TestVaultEditorPyCrypto, self).setUp()
def tearDown(self):
vault.HAS_CRYPTOGRAPHY = self.has_cryptography
super(TestVaultEditorPyCrypto, self).tearDown()

4
test/utils/shippable/other.sh
View file

@ -5,6 +5,10 @@ set -o pipefail
retry.py apt-get update -qq
retry.py apt-get install -qq \
shellcheck \
libssl-dev \
libffi-dev \
pip install cryptography
retry.py pip install tox --disable-pip-version-check

1
test/utils/tox/requirements.txt
View file

@ -11,6 +11,7 @@ unittest2
redis
python-memcached
python-systemd
cryptography
pycrypto
botocore
boto3