ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

Started reworking module_utils/basic unit tests (v2)

Browse source
This commit is contained in:
James Cammarata 2015-05-27 03:20:54 -05:00
parent f1ab1c48f4
commit 339a02c384
2 changed files with 199 additions and 259 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lib/ansible/module_utils/basic.py
View file

@ -930,7 +930,7 @@ class AnsibleModule(object):
for check in spec: for check in spec:
count = self._count_terms(check) count = self._count_terms(check)
if count > 1: if count > 1:
self.fail_json(msg="parameters are mutually exclusive: %s" % check) self.fail_json(msg="parameters are mutually exclusive: %s" % (check,))
def _check_required_one_of(self, spec): def _check_required_one_of(self, spec):
if spec is None: if spec is None:
@ -948,7 +948,7 @@ class AnsibleModule(object):
non_zero = [ c for c in counts if c > 0 ] non_zero = [ c for c in counts if c > 0 ]
if len(non_zero) > 0: if len(non_zero) > 0:
if 0 in counts: if 0 in counts:
self.fail_json(msg="parameters are required together: %s" % check) self.fail_json(msg="parameters are required together: %s" % (check,))
def _check_required_arguments(self): def _check_required_arguments(self):
''' ensure all required arguments are present ''' ''' ensure all required arguments are present '''

440
test/units/module_utils/test_basic.py
View file

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> # (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# #
# This file is part of Ansible # This file is part of Ansible
@ -16,204 +17,149 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# Make coding more python3-ish # Make coding more python3-ish
#from __future__ import (absolute_import, division, print_function)
from __future__ import (absolute_import, division) from __future__ import (absolute_import, division)
__metaclass__ = type __metaclass__ = type
import os import __builtin__
import tempfile
from nose.tools import timed
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock from ansible.compat.tests.mock import patch, MagicMock
from ansible.errors import *
from ansible.executor.module_common import modify_module
from ansible.module_utils.basic import heuristic_log_sanitize
from ansible.utils.hashing import checksum as utils_checksum
TEST_MODULE_DATA = """
from ansible.module_utils.basic import *
def get_module():
return AnsibleModule(
argument_spec = dict(),
supports_check_mode = True,
no_log = True,
)
get_module()
"""
class TestModuleUtilsBasic(unittest.TestCase): class TestModuleUtilsBasic(unittest.TestCase):
def cleanup_temp_file(self, fd, path):
try:
os.close(fd)
os.remove(path)
except:
pass
def cleanup_temp_dir(self, path):
try:
os.rmdir(path)
except:
pass
def setUp(self): def setUp(self):
# create a temporary file for the test module pass
# we're about to generate
self.tmp_fd, self.tmp_path = tempfile.mkstemp()
os.write(self.tmp_fd, TEST_MODULE_DATA)
# template the module code and eval it
module_data, module_style, shebang = modify_module(self.tmp_path, {})
d = {}
exec(module_data, d, d)
self.module = d['get_module']()
# module_utils/basic.py screws with CWD, let's save it and reset
self.cwd = os.getcwd()
def tearDown(self): def tearDown(self):
self.cleanup_temp_file(self.tmp_fd, self.tmp_path) pass
# Reset CWD back to what it was before basic.py changed it
os.chdir(self.cwd)
################################################################################# def test_module_utils_basic_imports(self):
# run_command() tests realimport = __builtin__.__import__
# test run_command with a string command def _mock_import(name, *args, **kwargs):
def test_run_command_string(self): if name == 'json':
(rc, out, err) = self.module.run_command("/bin/echo -n 'foo bar'") raise ImportError()
self.assertEqual(rc, 0) realimport(name, *args, **kwargs)
self.assertEqual(out, 'foo bar')
(rc, out, err) = self.module.run_command("/bin/echo -n 'foo bar'", use_unsafe_shell=True)
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar')
# test run_command with an array of args (with both use_unsafe_shell=True|False) with patch.object(__builtin__, '__import__', _mock_import, create=True) as m:
def test_run_command_args(self): m('ansible.module_utils.basic')
(rc, out, err) = self.module.run_command(['/bin/echo', '-n', "foo bar"]) __builtin__.__import__('ansible.module_utils.basic')
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar')
(rc, out, err) = self.module.run_command(['/bin/echo', '-n', "foo bar"], use_unsafe_shell=True)
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar')
# test run_command with leading environment variables def test_module_utils_basic_get_platform(self):
#@raises(SystemExit) with patch('platform.system', return_value='foo'):
def test_run_command_string_with_env_variables(self): from ansible.module_utils.basic import get_platform
self.assertRaises(SystemExit, self.module.run_command, 'FOO=bar /bin/echo -n "foo bar"') self.assertEqual(get_platform(), 'foo')
#@raises(SystemExit) def test_module_utils_basic_get_distribution(self):
def test_run_command_args_with_env_variables(self): from ansible.module_utils.basic import get_distribution
self.assertRaises(SystemExit, self.module.run_command, ['FOO=bar', '/bin/echo', '-n', 'foo bar'])
def test_run_command_string_unsafe_with_env_variables(self): with patch('platform.system', return_value='Foo'):
(rc, out, err) = self.module.run_command('FOO=bar /bin/echo -n "foo bar"', use_unsafe_shell=True) self.assertEqual(get_distribution(), None)
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar')
# test run_command with a command pipe (with both use_unsafe_shell=True|False) with patch('platform.system', return_value='Linux'):
def test_run_command_string_unsafe_with_pipe(self): with patch('platform.linux_distribution', return_value=("foo", "1", "One")):
(rc, out, err) = self.module.run_command('echo "foo bar" | cat', use_unsafe_shell=True) self.assertEqual(get_distribution(), "Foo")
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar\n')
# test run_command with a shell redirect in (with both use_unsafe_shell=True|False) with patch('os.path.isfile', return_value=True):
def test_run_command_string_unsafe_with_redirect_in(self): def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
(rc, out, err) = self.module.run_command('cat << EOF\nfoo bar\nEOF', use_unsafe_shell=True) if supported_dists != ():
self.assertEqual(rc, 0) return ("AmazonFooBar", "", "")
self.assertEqual(out, 'foo bar\n') else:
return ("", "", "")
# test run_command with a shell redirect out (with both use_unsafe_shell=True|False) with patch('platform.linux_distribution', side_effect=_dist):
def test_run_command_string_unsafe_with_redirect_out(self): self.assertEqual(get_distribution(), "Amazon")
tmp_fd, tmp_path = tempfile.mkstemp()
try:
(rc, out, err) = self.module.run_command('echo "foo bar" > %s' % tmp_path, use_unsafe_shell=True)
self.assertEqual(rc, 0)
self.assertTrue(os.path.exists(tmp_path))
checksum = utils_checksum(tmp_path)
self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec')
except:
raise
finally:
self.cleanup_temp_file(tmp_fd, tmp_path)
# test run_command with a double shell redirect out (append) (with both use_unsafe_shell=True|False) def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
def test_run_command_string_unsafe_with_double_redirect_out(self): if supported_dists != ():
tmp_fd, tmp_path = tempfile.mkstemp() return ("Bar", "2", "Two")
try: else:
(rc, out, err) = self.module.run_command('echo "foo bar" >> %s' % tmp_path, use_unsafe_shell=True) return ("", "", "")
self.assertEqual(rc, 0)
self.assertTrue(os.path.exists(tmp_path))
checksum = utils_checksum(tmp_path)
self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec')
except:
raise
finally:
self.cleanup_temp_file(tmp_fd, tmp_path)
# test run_command with data with patch('platform.linux_distribution', side_effect=_dist):
def test_run_command_string_with_data(self): self.assertEqual(get_distribution(), "OtherLinux")
(rc, out, err) = self.module.run_command('cat', data='foo bar')
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar\n')
# test run_command with binary data with patch('platform.linux_distribution', side_effect=Exception("boo")):
def test_run_command_string_with_binary_data(self): with patch('platform.dist', return_value=("bar", "2", "Two")):
(rc, out, err) = self.module.run_command('cat', data='\x41\x42\x43\x44', binary_data=True) self.assertEqual(get_distribution(), "Bar")
self.assertEqual(rc, 0)
self.assertEqual(out, 'ABCD')
# test run_command with a cwd set def test_module_utils_basic_get_distribution_version(self):
def test_run_command_string_with_cwd(self): from ansible.module_utils.basic import get_distribution_version
tmp_path = tempfile.mkdtemp()
try:
(rc, out, err) = self.module.run_command('pwd', cwd=tmp_path)
self.assertEqual(rc, 0)
self.assertTrue(os.path.exists(tmp_path))
self.assertEqual(out.strip(), os.path.realpath(tmp_path))
except:
raise
finally:
self.cleanup_temp_dir(tmp_path)
with patch('platform.system', return_value='Foo'):
self.assertEqual(get_distribution_version(), None)
class TestModuleUtilsBasicHelpers(unittest.TestCase): with patch('platform.system', return_value='Linux'):
''' Test some implementation details of AnsibleModule with patch('platform.linux_distribution', return_value=("foo", "1", "One")):
self.assertEqual(get_distribution_version(), "1")
Some pieces of AnsibleModule are implementation details but they have with patch('os.path.isfile', return_value=True):
potential cornercases that we need to check. Go ahead and test at def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
this level that the functions are behaving even though their API may if supported_dists != ():
change and we'd have to rewrite these tests so that we know that we return ("AmazonFooBar", "2", "")
need to check for those problems in any rewrite. else:
return ("", "", "")
In the future we might want to restructure higher level code to be with patch('platform.linux_distribution', side_effect=_dist):
friendlier to unittests so that we can test at the level that the public self.assertEqual(get_distribution_version(), "2")
is interacting with the APIs.
''' with patch('platform.linux_distribution', side_effect=Exception("boo")):
with patch('platform.dist', return_value=("bar", "3", "Three")):
self.assertEqual(get_distribution_version(), "3")
def test_module_utils_basic_load_platform_subclass(self):
class LinuxTest:
pass
class Foo(LinuxTest):
platform = "Linux"
distribution = None
class Bar(LinuxTest):
platform = "Linux"
distribution = "Bar"
from ansible.module_utils.basic import load_platform_subclass
# match just the platform class, not a specific distribution
with patch('ansible.module_utils.basic.get_platform', return_value="Linux"):
with patch('ansible.module_utils.basic.get_distribution', return_value=None):
self.assertIs(type(load_platform_subclass(LinuxTest)), Foo)
# match both the distribution and platform class
with patch('ansible.module_utils.basic.get_platform', return_value="Linux"):
with patch('ansible.module_utils.basic.get_distribution', return_value="Bar"):
self.assertIs(type(load_platform_subclass(LinuxTest)), Bar)
# if neither match, the fallback should be the top-level class
with patch('ansible.module_utils.basic.get_platform', return_value="Foo"):
with patch('ansible.module_utils.basic.get_distribution', return_value=None):
self.assertIs(type(load_platform_subclass(LinuxTest)), LinuxTest)
def test_module_utils_basic_json_dict_converters(self):
from ansible.module_utils.basic import json_dict_unicode_to_bytes, json_dict_bytes_to_unicode
test_data = dict(
item1 = u"Fóo",
item2 = [u"Bár", u"Bam"],
item3 = dict(sub1=u"Súb"),
item4 = (u"föo", u"bär", u"©"),
item5 = 42,
)
res = json_dict_unicode_to_bytes(test_data)
res2 = json_dict_bytes_to_unicode(res)
self.assertEqual(test_data, res2)
def test_module_utils_basic_heuristic_log_sanitize(self):
from ansible.module_utils.basic import heuristic_log_sanitize
MANY_RECORDS = 7000
URL_SECRET = 'http://username:pas:word@foo.com/data' URL_SECRET = 'http://username:pas:word@foo.com/data'
SSH_SECRET = 'username:pas:word@foo.com/data' SSH_SECRET = 'username:pas:word@foo.com/data'
def cleanup_temp_file(self, fd, path): def _gen_data(records, per_rec, top_level, secret_text):
try:
os.close(fd)
os.remove(path)
except:
pass
def cleanup_temp_dir(self, path):
try:
os.rmdir(path)
except:
pass
def _gen_data(self, records, per_rec, top_level, secret_text):
hostvars = {'hostvars': {}} hostvars = {'hostvars': {}}
for i in range(1, records, 1): for i in range(1, records, 1):
host_facts = {'host%s' % i: host_facts = {'host%s' % i:
@ -223,7 +169,6 @@ class TestModuleUtilsBasicHelpers(unittest.TestCase):
'tars': [], 'tars': [],
'versions': ['885.0']}, 'versions': ['885.0']},
}} }}
if per_rec: if per_rec:
host_facts['host%s' % i]['secret'] = secret_text host_facts['host%s' % i]['secret'] = secret_text
hostvars['hostvars'].update(host_facts) hostvars['hostvars'].update(host_facts)
@ -231,86 +176,8 @@ class TestModuleUtilsBasicHelpers(unittest.TestCase):
hostvars['secret'] = secret_text hostvars['secret'] = secret_text
return hostvars return hostvars
def setUp(self): url_data = repr(_gen_data(3, True, True, URL_SECRET))
self.many_url = repr(self._gen_data(self.MANY_RECORDS, True, True, ssh_data = repr(_gen_data(3, True, True, SSH_SECRET))
self.URL_SECRET))
self.many_ssh = repr(self._gen_data(self.MANY_RECORDS, True, True,
self.SSH_SECRET))
self.one_url = repr(self._gen_data(self.MANY_RECORDS, False, True,
self.URL_SECRET))
self.one_ssh = repr(self._gen_data(self.MANY_RECORDS, False, True,
self.SSH_SECRET))
self.zero_secrets = repr(self._gen_data(self.MANY_RECORDS, False,
False, ''))
self.few_url = repr(self._gen_data(2, True, True, self.URL_SECRET))
self.few_ssh = repr(self._gen_data(2, True, True, self.SSH_SECRET))
# create a temporary file for the test module
# we're about to generate
self.tmp_fd, self.tmp_path = tempfile.mkstemp()
os.write(self.tmp_fd, TEST_MODULE_DATA)
# template the module code and eval it
module_data, module_style, shebang = modify_module(self.tmp_path, {})
d = {}
exec(module_data, d, d)
self.module = d['get_module']()
# module_utils/basic.py screws with CWD, let's save it and reset
self.cwd = os.getcwd()
def tearDown(self):
self.cleanup_temp_file(self.tmp_fd, self.tmp_path)
# Reset CWD back to what it was before basic.py changed it
os.chdir(self.cwd)
#################################################################################
#
# Speed tests
#
# Previously, we used regexes which had some pathologically slow cases for
# parameters with large amounts of data with many ':' but no '@'. The
# present function gets slower when there are many replacements so we may
# want to explore regexes in the future (for the speed when substituting
# or flexibility). These speed tests will hopefully tell us if we're
# introducing code that has cases that are simply too slow.
#
# Some regex notes:
# * re.sub() is faster than re.match() + str.join().
# * We may be able to detect a large number of '@' symbols and then use
# a regex else use the present function.
#@timed(5)
#def test_log_sanitize_speed_many_url(self):
# heuristic_log_sanitize(self.many_url)
#@timed(5)
#def test_log_sanitize_speed_many_ssh(self):
# heuristic_log_sanitize(self.many_ssh)
#@timed(5)
#def test_log_sanitize_speed_one_url(self):
# heuristic_log_sanitize(self.one_url)
#@timed(5)
#def test_log_sanitize_speed_one_ssh(self):
# heuristic_log_sanitize(self.one_ssh)
#@timed(5)
#def test_log_sanitize_speed_zero_secrets(self):
# heuristic_log_sanitize(self.zero_secrets)
#
# Test that the password obfuscation sanitizes somewhat cleanly.
#
def test_log_sanitize_correctness(self):
url_data = repr(self._gen_data(3, True, True, self.URL_SECRET))
ssh_data = repr(self._gen_data(3, True, True, self.SSH_SECRET))
url_output = heuristic_log_sanitize(url_data) url_output = heuristic_log_sanitize(url_data)
ssh_output = heuristic_log_sanitize(ssh_data) ssh_output = heuristic_log_sanitize(ssh_data)
@ -349,7 +216,80 @@ class TestModuleUtilsBasicHelpers(unittest.TestCase):
# python2.6 or less's unittest # python2.6 or less's unittest
self.assertTrue(":********@foo.com/data'" in ssh_output, '%s is not present in %s' % (":********@foo.com/data'", ssh_output)) self.assertTrue(":********@foo.com/data'" in ssh_output, '%s is not present in %s' % (":********@foo.com/data'", ssh_output))
# The overzealous-ness here may lead to us changing the algorithm in
# the future. We could make it consume less of the data (with the def test_module_utils_basic_ansible_module_creation(self):
# possibility of leaving partial passwords exposed) and encourage from ansible.module_utils import basic
# people to use no_log instead of relying on this obfuscation.
basic.MODULE_COMPLEX_ARGS = '{}'
am = basic.AnsibleModule(
argument_spec=dict(),
)
arg_spec = dict(
foo = dict(required=True),
bar = dict(),
bam = dict(),
baz = dict(),
)
mut_ex = (('bar', 'bam'),)
req_to = (('bam', 'baz'),)
# should test ok
basic.MODULE_COMPLEX_ARGS = '{"foo":"hello"}'
am = basic.AnsibleModule(
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
# fail, because a required param was not specified
basic.MODULE_COMPLEX_ARGS = '{}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
# fail because of mutually exclusive parameters
basic.MODULE_COMPLEX_ARGS = '{"foo":"hello", "bar": "bad", "bam": "bad"}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
# fail because a param required due to another param was not specified
basic.MODULE_COMPLEX_ARGS = '{"bam":"bad"}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
def test_module_utils_basic_get_module_path(self):
from ansible.module_utils.basic import get_module_path
with patch('os.path.realpath', return_value='/path/to/foo/'):
self.assertEqual(get_module_path(), '/path/to/foo')