From 5775739437f28077ebcaaf0686eed4c7d622c02c Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Fri, 11 Mar 2016 21:59:06 +0100 Subject: [PATCH 001/133] Moving samples/ out of the root into test/ Since samples/ is used for v2 testing, it fits better into the existing test/ structure. --- {samples => test/samples}/README.md | 0 {samples => test/samples}/common_include.yml | 0 {samples => test/samples}/hosts | 0 {samples => test/samples}/ignore_errors.yml | 0 {samples => test/samples}/include.yml | 0 {samples => test/samples}/included_playbook.yml | 0 {samples => test/samples}/inv_lg | 0 {samples => test/samples}/inv_md | 0 {samples => test/samples}/inv_sm | 0 {samples => test/samples}/l1_include.yml | 0 {samples => test/samples}/l2_include.yml | 0 {samples => test/samples}/l3_include.yml | 0 {samples => test/samples}/localhost_include.yml | 0 {samples => test/samples}/localhosts | 0 {samples => test/samples}/lookup_file.yml | 0 {samples => test/samples}/lookup_password.yml | 0 {samples => test/samples}/lookup_pipe.py | 0 {samples => test/samples}/lookup_template.yml | 0 {samples => test/samples}/multi.py | 0 {samples => test/samples}/multi_queues.py | 0 {samples => test/samples}/roles/common/meta/main.yml | 0 {samples => test/samples}/roles/common/tasks/main.yml | 0 {samples => test/samples}/roles/role_a/meta/main.yml | 0 {samples => test/samples}/roles/role_a/tasks/main.yml | 0 {samples => test/samples}/roles/role_b/meta/main.yml | 0 {samples => test/samples}/roles/role_b/tasks/main.yml | 0 {samples => test/samples}/roles/test_become_r1/meta/main.yml | 0 {samples => test/samples}/roles/test_become_r1/tasks/main.yml | 0 {samples => test/samples}/roles/test_become_r2/meta/main.yml | 0 {samples => test/samples}/roles/test_become_r2/tasks/main.yml | 0 {samples => test/samples}/roles/test_role/meta/main.yml | 0 {samples => test/samples}/roles/test_role/tasks/main.yml | 0 {samples => test/samples}/roles/test_role_dep/tasks/main.yml | 0 {samples => test/samples}/src | 0 {samples => test/samples}/template.j2 | 0 {samples => test/samples}/test_become.yml | 0 {samples => test/samples}/test_big_debug.yml | 0 {samples => test/samples}/test_big_ping.yml | 0 {samples => test/samples}/test_block.yml | 0 {samples => test/samples}/test_blocks_of_blocks.yml | 0 {samples => test/samples}/test_fact_gather.yml | 0 {samples => test/samples}/test_free.yml | 0 {samples => test/samples}/test_include.yml | 0 {samples => test/samples}/test_pb.yml | 0 {samples => test/samples}/test_play_failure.yml | 0 {samples => test/samples}/test_playbook.include | 0 {samples => test/samples}/test_role.yml | 0 {samples => test/samples}/test_roles_complex.yml | 0 {samples => test/samples}/test_run_once.yml | 0 {samples => test/samples}/test_sudo.yml | 0 {samples => test/samples}/test_tags.yml | 0 {samples => test/samples}/testing/extra_vars.yml | 0 {samples => test/samples}/testing/frag1 | 0 {samples => test/samples}/testing/frag2 | 0 {samples => test/samples}/testing/frag3 | 0 {samples => test/samples}/testing/vars.yml | 0 {samples => test/samples}/with_dict.yml | 0 {samples => test/samples}/with_env.yml | 0 {samples => test/samples}/with_fileglob.yml | 0 {samples => test/samples}/with_first_found.yml | 0 {samples => test/samples}/with_flattened.yml | 0 {samples => test/samples}/with_indexed_items.yml | 0 {samples => test/samples}/with_items.yml | 0 {samples => test/samples}/with_lines.yml | 0 {samples => test/samples}/with_nested.yml | 0 {samples => test/samples}/with_random_choice.yml | 0 {samples => test/samples}/with_sequence.yml | 0 {samples => test/samples}/with_subelements.yml | 0 {samples => test/samples}/with_together.yml | 0 69 files changed, 0 insertions(+), 0 deletions(-) rename {samples => test/samples}/README.md (100%) rename {samples => test/samples}/common_include.yml (100%) rename {samples => test/samples}/hosts (100%) rename {samples => test/samples}/ignore_errors.yml (100%) rename {samples => test/samples}/include.yml (100%) rename {samples => test/samples}/included_playbook.yml (100%) rename {samples => test/samples}/inv_lg (100%) rename {samples => test/samples}/inv_md (100%) rename {samples => test/samples}/inv_sm (100%) rename {samples => test/samples}/l1_include.yml (100%) rename {samples => test/samples}/l2_include.yml (100%) rename {samples => test/samples}/l3_include.yml (100%) rename {samples => test/samples}/localhost_include.yml (100%) rename {samples => test/samples}/localhosts (100%) rename {samples => test/samples}/lookup_file.yml (100%) rename {samples => test/samples}/lookup_password.yml (100%) rename {samples => test/samples}/lookup_pipe.py (100%) rename {samples => test/samples}/lookup_template.yml (100%) rename {samples => test/samples}/multi.py (100%) rename {samples => test/samples}/multi_queues.py (100%) rename {samples => test/samples}/roles/common/meta/main.yml (100%) rename {samples => test/samples}/roles/common/tasks/main.yml (100%) rename {samples => test/samples}/roles/role_a/meta/main.yml (100%) rename {samples => test/samples}/roles/role_a/tasks/main.yml (100%) rename {samples => test/samples}/roles/role_b/meta/main.yml (100%) rename {samples => test/samples}/roles/role_b/tasks/main.yml (100%) rename {samples => test/samples}/roles/test_become_r1/meta/main.yml (100%) rename {samples => test/samples}/roles/test_become_r1/tasks/main.yml (100%) rename {samples => test/samples}/roles/test_become_r2/meta/main.yml (100%) rename {samples => test/samples}/roles/test_become_r2/tasks/main.yml (100%) rename {samples => test/samples}/roles/test_role/meta/main.yml (100%) rename {samples => test/samples}/roles/test_role/tasks/main.yml (100%) rename {samples => test/samples}/roles/test_role_dep/tasks/main.yml (100%) rename {samples => test/samples}/src (100%) rename {samples => test/samples}/template.j2 (100%) rename {samples => test/samples}/test_become.yml (100%) rename {samples => test/samples}/test_big_debug.yml (100%) rename {samples => test/samples}/test_big_ping.yml (100%) rename {samples => test/samples}/test_block.yml (100%) rename {samples => test/samples}/test_blocks_of_blocks.yml (100%) rename {samples => test/samples}/test_fact_gather.yml (100%) rename {samples => test/samples}/test_free.yml (100%) rename {samples => test/samples}/test_include.yml (100%) rename {samples => test/samples}/test_pb.yml (100%) rename {samples => test/samples}/test_play_failure.yml (100%) rename {samples => test/samples}/test_playbook.include (100%) rename {samples => test/samples}/test_role.yml (100%) rename {samples => test/samples}/test_roles_complex.yml (100%) rename {samples => test/samples}/test_run_once.yml (100%) rename {samples => test/samples}/test_sudo.yml (100%) rename {samples => test/samples}/test_tags.yml (100%) rename {samples => test/samples}/testing/extra_vars.yml (100%) rename {samples => test/samples}/testing/frag1 (100%) rename {samples => test/samples}/testing/frag2 (100%) rename {samples => test/samples}/testing/frag3 (100%) rename {samples => test/samples}/testing/vars.yml (100%) rename {samples => test/samples}/with_dict.yml (100%) rename {samples => test/samples}/with_env.yml (100%) rename {samples => test/samples}/with_fileglob.yml (100%) rename {samples => test/samples}/with_first_found.yml (100%) rename {samples => test/samples}/with_flattened.yml (100%) rename {samples => test/samples}/with_indexed_items.yml (100%) rename {samples => test/samples}/with_items.yml (100%) rename {samples => test/samples}/with_lines.yml (100%) rename {samples => test/samples}/with_nested.yml (100%) rename {samples => test/samples}/with_random_choice.yml (100%) rename {samples => test/samples}/with_sequence.yml (100%) rename {samples => test/samples}/with_subelements.yml (100%) rename {samples => test/samples}/with_together.yml (100%) diff --git a/samples/README.md b/test/samples/README.md similarity index 100% rename from samples/README.md rename to test/samples/README.md diff --git a/samples/common_include.yml b/test/samples/common_include.yml similarity index 100% rename from samples/common_include.yml rename to test/samples/common_include.yml diff --git a/samples/hosts b/test/samples/hosts similarity index 100% rename from samples/hosts rename to test/samples/hosts diff --git a/samples/ignore_errors.yml b/test/samples/ignore_errors.yml similarity index 100% rename from samples/ignore_errors.yml rename to test/samples/ignore_errors.yml diff --git a/samples/include.yml b/test/samples/include.yml similarity index 100% rename from samples/include.yml rename to test/samples/include.yml diff --git a/samples/included_playbook.yml b/test/samples/included_playbook.yml similarity index 100% rename from samples/included_playbook.yml rename to test/samples/included_playbook.yml diff --git a/samples/inv_lg b/test/samples/inv_lg similarity index 100% rename from samples/inv_lg rename to test/samples/inv_lg diff --git a/samples/inv_md b/test/samples/inv_md similarity index 100% rename from samples/inv_md rename to test/samples/inv_md diff --git a/samples/inv_sm b/test/samples/inv_sm similarity index 100% rename from samples/inv_sm rename to test/samples/inv_sm diff --git a/samples/l1_include.yml b/test/samples/l1_include.yml similarity index 100% rename from samples/l1_include.yml rename to test/samples/l1_include.yml diff --git a/samples/l2_include.yml b/test/samples/l2_include.yml similarity index 100% rename from samples/l2_include.yml rename to test/samples/l2_include.yml diff --git a/samples/l3_include.yml b/test/samples/l3_include.yml similarity index 100% rename from samples/l3_include.yml rename to test/samples/l3_include.yml diff --git a/samples/localhost_include.yml b/test/samples/localhost_include.yml similarity index 100% rename from samples/localhost_include.yml rename to test/samples/localhost_include.yml diff --git a/samples/localhosts b/test/samples/localhosts similarity index 100% rename from samples/localhosts rename to test/samples/localhosts diff --git a/samples/lookup_file.yml b/test/samples/lookup_file.yml similarity index 100% rename from samples/lookup_file.yml rename to test/samples/lookup_file.yml diff --git a/samples/lookup_password.yml b/test/samples/lookup_password.yml similarity index 100% rename from samples/lookup_password.yml rename to test/samples/lookup_password.yml diff --git a/samples/lookup_pipe.py b/test/samples/lookup_pipe.py similarity index 100% rename from samples/lookup_pipe.py rename to test/samples/lookup_pipe.py diff --git a/samples/lookup_template.yml b/test/samples/lookup_template.yml similarity index 100% rename from samples/lookup_template.yml rename to test/samples/lookup_template.yml diff --git a/samples/multi.py b/test/samples/multi.py similarity index 100% rename from samples/multi.py rename to test/samples/multi.py diff --git a/samples/multi_queues.py b/test/samples/multi_queues.py similarity index 100% rename from samples/multi_queues.py rename to test/samples/multi_queues.py diff --git a/samples/roles/common/meta/main.yml b/test/samples/roles/common/meta/main.yml similarity index 100% rename from samples/roles/common/meta/main.yml rename to test/samples/roles/common/meta/main.yml diff --git a/samples/roles/common/tasks/main.yml b/test/samples/roles/common/tasks/main.yml similarity index 100% rename from samples/roles/common/tasks/main.yml rename to test/samples/roles/common/tasks/main.yml diff --git a/samples/roles/role_a/meta/main.yml b/test/samples/roles/role_a/meta/main.yml similarity index 100% rename from samples/roles/role_a/meta/main.yml rename to test/samples/roles/role_a/meta/main.yml diff --git a/samples/roles/role_a/tasks/main.yml b/test/samples/roles/role_a/tasks/main.yml similarity index 100% rename from samples/roles/role_a/tasks/main.yml rename to test/samples/roles/role_a/tasks/main.yml diff --git a/samples/roles/role_b/meta/main.yml b/test/samples/roles/role_b/meta/main.yml similarity index 100% rename from samples/roles/role_b/meta/main.yml rename to test/samples/roles/role_b/meta/main.yml diff --git a/samples/roles/role_b/tasks/main.yml b/test/samples/roles/role_b/tasks/main.yml similarity index 100% rename from samples/roles/role_b/tasks/main.yml rename to test/samples/roles/role_b/tasks/main.yml diff --git a/samples/roles/test_become_r1/meta/main.yml b/test/samples/roles/test_become_r1/meta/main.yml similarity index 100% rename from samples/roles/test_become_r1/meta/main.yml rename to test/samples/roles/test_become_r1/meta/main.yml diff --git a/samples/roles/test_become_r1/tasks/main.yml b/test/samples/roles/test_become_r1/tasks/main.yml similarity index 100% rename from samples/roles/test_become_r1/tasks/main.yml rename to test/samples/roles/test_become_r1/tasks/main.yml diff --git a/samples/roles/test_become_r2/meta/main.yml b/test/samples/roles/test_become_r2/meta/main.yml similarity index 100% rename from samples/roles/test_become_r2/meta/main.yml rename to test/samples/roles/test_become_r2/meta/main.yml diff --git a/samples/roles/test_become_r2/tasks/main.yml b/test/samples/roles/test_become_r2/tasks/main.yml similarity index 100% rename from samples/roles/test_become_r2/tasks/main.yml rename to test/samples/roles/test_become_r2/tasks/main.yml diff --git a/samples/roles/test_role/meta/main.yml b/test/samples/roles/test_role/meta/main.yml similarity index 100% rename from samples/roles/test_role/meta/main.yml rename to test/samples/roles/test_role/meta/main.yml diff --git a/samples/roles/test_role/tasks/main.yml b/test/samples/roles/test_role/tasks/main.yml similarity index 100% rename from samples/roles/test_role/tasks/main.yml rename to test/samples/roles/test_role/tasks/main.yml diff --git a/samples/roles/test_role_dep/tasks/main.yml b/test/samples/roles/test_role_dep/tasks/main.yml similarity index 100% rename from samples/roles/test_role_dep/tasks/main.yml rename to test/samples/roles/test_role_dep/tasks/main.yml diff --git a/samples/src b/test/samples/src similarity index 100% rename from samples/src rename to test/samples/src diff --git a/samples/template.j2 b/test/samples/template.j2 similarity index 100% rename from samples/template.j2 rename to test/samples/template.j2 diff --git a/samples/test_become.yml b/test/samples/test_become.yml similarity index 100% rename from samples/test_become.yml rename to test/samples/test_become.yml diff --git a/samples/test_big_debug.yml b/test/samples/test_big_debug.yml similarity index 100% rename from samples/test_big_debug.yml rename to test/samples/test_big_debug.yml diff --git a/samples/test_big_ping.yml b/test/samples/test_big_ping.yml similarity index 100% rename from samples/test_big_ping.yml rename to test/samples/test_big_ping.yml diff --git a/samples/test_block.yml b/test/samples/test_block.yml similarity index 100% rename from samples/test_block.yml rename to test/samples/test_block.yml diff --git a/samples/test_blocks_of_blocks.yml b/test/samples/test_blocks_of_blocks.yml similarity index 100% rename from samples/test_blocks_of_blocks.yml rename to test/samples/test_blocks_of_blocks.yml diff --git a/samples/test_fact_gather.yml b/test/samples/test_fact_gather.yml similarity index 100% rename from samples/test_fact_gather.yml rename to test/samples/test_fact_gather.yml diff --git a/samples/test_free.yml b/test/samples/test_free.yml similarity index 100% rename from samples/test_free.yml rename to test/samples/test_free.yml diff --git a/samples/test_include.yml b/test/samples/test_include.yml similarity index 100% rename from samples/test_include.yml rename to test/samples/test_include.yml diff --git a/samples/test_pb.yml b/test/samples/test_pb.yml similarity index 100% rename from samples/test_pb.yml rename to test/samples/test_pb.yml diff --git a/samples/test_play_failure.yml b/test/samples/test_play_failure.yml similarity index 100% rename from samples/test_play_failure.yml rename to test/samples/test_play_failure.yml diff --git a/samples/test_playbook.include b/test/samples/test_playbook.include similarity index 100% rename from samples/test_playbook.include rename to test/samples/test_playbook.include diff --git a/samples/test_role.yml b/test/samples/test_role.yml similarity index 100% rename from samples/test_role.yml rename to test/samples/test_role.yml diff --git a/samples/test_roles_complex.yml b/test/samples/test_roles_complex.yml similarity index 100% rename from samples/test_roles_complex.yml rename to test/samples/test_roles_complex.yml diff --git a/samples/test_run_once.yml b/test/samples/test_run_once.yml similarity index 100% rename from samples/test_run_once.yml rename to test/samples/test_run_once.yml diff --git a/samples/test_sudo.yml b/test/samples/test_sudo.yml similarity index 100% rename from samples/test_sudo.yml rename to test/samples/test_sudo.yml diff --git a/samples/test_tags.yml b/test/samples/test_tags.yml similarity index 100% rename from samples/test_tags.yml rename to test/samples/test_tags.yml diff --git a/samples/testing/extra_vars.yml b/test/samples/testing/extra_vars.yml similarity index 100% rename from samples/testing/extra_vars.yml rename to test/samples/testing/extra_vars.yml diff --git a/samples/testing/frag1 b/test/samples/testing/frag1 similarity index 100% rename from samples/testing/frag1 rename to test/samples/testing/frag1 diff --git a/samples/testing/frag2 b/test/samples/testing/frag2 similarity index 100% rename from samples/testing/frag2 rename to test/samples/testing/frag2 diff --git a/samples/testing/frag3 b/test/samples/testing/frag3 similarity index 100% rename from samples/testing/frag3 rename to test/samples/testing/frag3 diff --git a/samples/testing/vars.yml b/test/samples/testing/vars.yml similarity index 100% rename from samples/testing/vars.yml rename to test/samples/testing/vars.yml diff --git a/samples/with_dict.yml b/test/samples/with_dict.yml similarity index 100% rename from samples/with_dict.yml rename to test/samples/with_dict.yml diff --git a/samples/with_env.yml b/test/samples/with_env.yml similarity index 100% rename from samples/with_env.yml rename to test/samples/with_env.yml diff --git a/samples/with_fileglob.yml b/test/samples/with_fileglob.yml similarity index 100% rename from samples/with_fileglob.yml rename to test/samples/with_fileglob.yml diff --git a/samples/with_first_found.yml b/test/samples/with_first_found.yml similarity index 100% rename from samples/with_first_found.yml rename to test/samples/with_first_found.yml diff --git a/samples/with_flattened.yml b/test/samples/with_flattened.yml similarity index 100% rename from samples/with_flattened.yml rename to test/samples/with_flattened.yml diff --git a/samples/with_indexed_items.yml b/test/samples/with_indexed_items.yml similarity index 100% rename from samples/with_indexed_items.yml rename to test/samples/with_indexed_items.yml diff --git a/samples/with_items.yml b/test/samples/with_items.yml similarity index 100% rename from samples/with_items.yml rename to test/samples/with_items.yml diff --git a/samples/with_lines.yml b/test/samples/with_lines.yml similarity index 100% rename from samples/with_lines.yml rename to test/samples/with_lines.yml diff --git a/samples/with_nested.yml b/test/samples/with_nested.yml similarity index 100% rename from samples/with_nested.yml rename to test/samples/with_nested.yml diff --git a/samples/with_random_choice.yml b/test/samples/with_random_choice.yml similarity index 100% rename from samples/with_random_choice.yml rename to test/samples/with_random_choice.yml diff --git a/samples/with_sequence.yml b/test/samples/with_sequence.yml similarity index 100% rename from samples/with_sequence.yml rename to test/samples/with_sequence.yml diff --git a/samples/with_subelements.yml b/test/samples/with_subelements.yml similarity index 100% rename from samples/with_subelements.yml rename to test/samples/with_subelements.yml diff --git a/samples/with_together.yml b/test/samples/with_together.yml similarity index 100% rename from samples/with_together.yml rename to test/samples/with_together.yml From bce79c67c3657ccf24bea9685eb158c5aab70e2e Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 6 Apr 2016 10:27:24 -0500 Subject: [PATCH 002/133] Cascade ssh_*args configurations in synchronize instead of limiting to just ssh_args. See https://github.com/ansible/ansible-modules-core/issues/3370 --- lib/ansible/plugins/action/synchronize.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/ansible/plugins/action/synchronize.py b/lib/ansible/plugins/action/synchronize.py index 0a74ed2d95..33b7a89fa1 100644 --- a/lib/ansible/plugins/action/synchronize.py +++ b/lib/ansible/plugins/action/synchronize.py @@ -318,7 +318,12 @@ class ActionModule(ActionBase): self._task.args['rsync_path'] = '"%s"' % rsync_path if use_ssh_args: - self._task.args['ssh_args'] = C.ANSIBLE_SSH_ARGS + ssh_args = [ + getattr(self._play_context, 'ssh_args', ''), + getattr(self._play_context, 'ssh_common_args', ''), + getattr(self._play_context, 'ssh_extra_args', ''), + ] + self._task.args['ssh_args'] = ' '.join([a for a in ssh_args if a]) # run the module and store the result result.update(self._execute_module('synchronize', task_vars=task_vars)) From 9278591758fa81022bb77d30f9e9cb92aff844f4 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 26 Mar 2015 21:38:18 -0700 Subject: [PATCH 003/133] add cobbler api authentication options add cobbler api authentication options: username and password, which can be provided if authentication is enabled or cobbler api is behind a proxy that needs authentication. --- contrib/inventory/cobbler.ini | 4 ++++ contrib/inventory/cobbler.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/contrib/inventory/cobbler.ini b/contrib/inventory/cobbler.ini index b7b63595de..2dc8cd3379 100644 --- a/contrib/inventory/cobbler.ini +++ b/contrib/inventory/cobbler.ini @@ -5,6 +5,10 @@ host = http://PATH_TO_COBBLER_SERVER/cobbler_api +# If API needs authentication add 'username' and 'password' options here. +#username = foo +#password = bar + # API calls to Cobbler can be slow. For this reason, we cache the results of an API # call. Set this to the path you want cache files to be written to. Two files # will be written to this directory: diff --git a/contrib/inventory/cobbler.py b/contrib/inventory/cobbler.py index b5fcdeacbb..89f9bf75b3 100755 --- a/contrib/inventory/cobbler.py +++ b/contrib/inventory/cobbler.py @@ -120,6 +120,9 @@ class CobblerInventory(object): def _connect(self): if not self.conn: self.conn = xmlrpclib.Server(self.cobbler_host, allow_none=True) + self.token = None + if self.cobbler_username is not None: + self.token = self.conn.login(self.cobbler_username, self.cobbler_password) def is_cache_valid(self): """ Determines if the cache files have expired, or if it is still valid """ @@ -140,6 +143,12 @@ class CobblerInventory(object): config.read(os.path.dirname(os.path.realpath(__file__)) + '/cobbler.ini') self.cobbler_host = config.get('cobbler', 'host') + self.cobbler_username = None + self.cobbler_password = None + if config.has_option('cobbler', 'username'): + self.cobbler_username = config.get('cobbler', 'username') + if config.has_option('cobbler', 'password'): + self.cobbler_password = config.get('cobbler', 'password') # Cache related cache_path = config.get('cobbler', 'cache_path') @@ -163,8 +172,10 @@ class CobblerInventory(object): self._connect() self.groups = dict() self.hosts = dict() - - data = self.conn.get_systems() + if self.token is not None: + data = self.conn.get_systems(self.token) + else: + data = self.conn.get_systems() for host in data: # Get the FQDN for the host and add it to the right groups From c43ea832754d0640dcf7ba9ae2cb2759476a2ea2 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 27 Apr 2016 14:41:00 -0400 Subject: [PATCH 004/133] fixes #15496 changed to using OrderedDict to preserve order of lines --- lib/ansible/module_utils/netcfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/netcfg.py b/lib/ansible/module_utils/netcfg.py index ff1d138c23..68607f525c 100644 --- a/lib/ansible/module_utils/netcfg.py +++ b/lib/ansible/module_utils/netcfg.py @@ -229,7 +229,7 @@ class NetworkConfig(object): if self._device_os == 'junos': return updates - diffs = dict() + diffs = collections.OrderedDict() for update in updates: if replace == 'block' and update.parents: update = update.parents[-1] From ae9ddf0c1c44f3c4e9650a74cf06033a5e35ecaf Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 3 May 2016 10:12:02 -0700 Subject: [PATCH 005/133] Submodule updates to fix documentation --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index bb9572ca86..9db1233521 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit bb9572ca861ff35ce85a34087be892e25a268391 +Subproject commit 9db123352168e81f8c81e95225a64663d2724d5b diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 7fd4180857..675d778b50 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 7fd4180857f856a59792724e02e95dd99c067083 +Subproject commit 675d778b50257a72290a68f4f644a4f630e85ce4 From 606d35b8a611eea77cf10895122d4fba3f64a4b2 Mon Sep 17 00:00:00 2001 From: John R Barker Date: Tue, 3 May 2016 19:18:48 +0100 Subject: [PATCH 006/133] Track build times (#15708) See if https://buildtimetrend.herokuapp.com/ gives us any extra insights into how we can speed up builds --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c3d250b9e5..9731446994 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,3 +38,6 @@ notifications: on_failure: always skip_join: true nick: ansibletravis + webhooks: + # trigger Buildtime Trend Service to parse Travis CI log + - https://buildtimetrend.herokuapp.com/travis From cf62a62b83a461658c2161ad7a6c433e33308515 Mon Sep 17 00:00:00 2001 From: Robin Roth Date: Wed, 4 May 2016 00:09:26 +0200 Subject: [PATCH 007/133] use userdir module as example instead of alias (#15540) * alias module is very basic and removing it leads to the suse default config failing * future improvements might test different modules and the effect of them being removed --- .../test_apache2_module/tasks/actualtest.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/integration/roles/test_apache2_module/tasks/actualtest.yml b/test/integration/roles/test_apache2_module/tasks/actualtest.yml index a5f3a40a3c..5b02a1c2ff 100644 --- a/test/integration/roles/test_apache2_module/tasks/actualtest.yml +++ b/test/integration/roles/test_apache2_module/tasks/actualtest.yml @@ -21,11 +21,11 @@ zypper: name=apache2 state=present when: "ansible_os_family == 'Suse'" -- name: disable alias module - apache2_module: name=alias state=absent +- name: disable userdir module + apache2_module: name=userdir state=absent -- name: disable alias module, second run - apache2_module: name=alias state=absent +- name: disable userdir module, second run + apache2_module: name=userdir state=absent register: disable - name: ensure apache2_module is idempotent @@ -33,8 +33,8 @@ that: - 'not disable.changed' -- name: enable alias module - apache2_module: name=alias state=present +- name: enable userdir module + apache2_module: name=userdir state=present register: enable - name: ensure changed on successful enable @@ -42,8 +42,8 @@ that: - 'enable.changed' -- name: enable alias module, second run - apache2_module: name=alias state=present +- name: enable userdir module, second run + apache2_module: name=userdir state=present register: enabletwo - name: ensure apache2_module is idempotent @@ -51,8 +51,8 @@ that: - 'not enabletwo.changed' -- name: disable alias module, final run - apache2_module: name=alias state=absent +- name: disable userdir module, final run + apache2_module: name=userdir state=absent register: disablefinal - name: ensure changed on successful disable From c20d1fced7c5ee786d298da6a08fb60bd75cf0f3 Mon Sep 17 00:00:00 2001 From: Rene Moser Date: Sat, 23 Apr 2016 22:43:11 +0200 Subject: [PATCH 008/133] tests: fix tests on Debian 8 --- .../roles/setup_postgresql_db/tasks/main.yml | 6 +++--- .../roles/setup_postgresql_db/vars/Debian-8.yml | 10 ++++++++++ test/integration/roles/test_apt/tasks/apt.yml | 10 +++++----- .../roles/test_apt_repository/tasks/main.yml | 2 +- test/integration/roles/test_service/tasks/main.yml | 2 +- .../roles/test_service/tasks/systemd_setup.yml | 8 ++++---- 6 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 test/integration/roles/setup_postgresql_db/vars/Debian-8.yml diff --git a/test/integration/roles/setup_postgresql_db/tasks/main.yml b/test/integration/roles/setup_postgresql_db/tasks/main.yml index 38bcd50dcc..c49eede664 100644 --- a/test/integration/roles/setup_postgresql_db/tasks/main.yml +++ b/test/integration/roles/setup_postgresql_db/tasks/main.yml @@ -52,15 +52,15 @@ loop_var: postgresql_package_item when: ansible_pkg_mgr == 'apt' -- name: Initialize postgres (systemd) +- name: Initialize postgres (RedHat systemd) command: postgresql-setup initdb when: ansible_distribution == "Fedora" or (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 7) -- name: Initialize postgres (sysv) +- name: Initialize postgres (RedHat sysv) command: /sbin/service postgresql initdb when: ansible_os_family == "RedHat" and ansible_distribution_major_version|int <= 6 -- name: Iniitalize postgres (upstart) +- name: Iniitalize postgres (Debian) command: /usr/bin/pg_createcluster {{ pg_ver }} main # Sometimes package install creates the db cluster, sometimes this step is needed ignore_errors: True diff --git a/test/integration/roles/setup_postgresql_db/vars/Debian-8.yml b/test/integration/roles/setup_postgresql_db/vars/Debian-8.yml new file mode 100644 index 0000000000..f829bd1996 --- /dev/null +++ b/test/integration/roles/setup_postgresql_db/vars/Debian-8.yml @@ -0,0 +1,10 @@ +postgresql_service: "postgresql" + +postgresql_packages: + - "postgresql" + - "postgresql-common" + - "python-psycopg2" + +pg_hba_location: "/etc/postgresql/9.4/main/pg_hba.conf" +pg_dir: "/var/lib/postgresql/9.4/main" +pg_ver: 9.4 diff --git a/test/integration/roles/test_apt/tasks/apt.yml b/test/integration/roles/test_apt/tasks/apt.yml index 5d331b192d..201404f647 100644 --- a/test/integration/roles/test_apt/tasks/apt.yml +++ b/test/integration/roles/test_apt/tasks/apt.yml @@ -21,7 +21,7 @@ register: apt_result - name: check hello with dpkg - shell: dpkg --get-selections | fgrep hello + shell: dpkg-query -l hello failed_when: False register: dpkg_result @@ -47,7 +47,7 @@ register: apt_result - name: check hello with dpkg - shell: dpkg --get-selections | fgrep hello + shell: dpkg-query -l hello failed_when: False register: dpkg_result @@ -89,7 +89,7 @@ register: apt_result - name: check hello with wildcard with dpkg - shell: dpkg --get-selections | fgrep hello + shell: dpkg-query -l hello failed_when: False register: dpkg_result @@ -103,10 +103,10 @@ - "dpkg_result.rc == 0" - name: check hello version - shell: dpkg -s hello | grep Version | sed -r 's/Version:\s+([a-zA-Z0-9.-]+)\s*$/\1/' + shell: dpkg -s hello | grep Version | awk '{print $2}' register: hello_version - name: check hello architecture - shell: dpkg -s hello | grep Architecture | sed -r 's/Architecture:\s+([a-zA-Z0-9.-]+)\s*$/\1/' + shell: dpkg -s hello | grep Architecture | awk '{print $2}' register: hello_architecture - name: uninstall hello with apt diff --git a/test/integration/roles/test_apt_repository/tasks/main.yml b/test/integration/roles/test_apt_repository/tasks/main.yml index 8a16a061bd..38ae8f0447 100644 --- a/test/integration/roles/test_apt_repository/tasks/main.yml +++ b/test/integration/roles/test_apt_repository/tasks/main.yml @@ -17,5 +17,5 @@ # along with Ansible. If not, see . - include: 'apt.yml' - when: ansible_distribution in ('Ubuntu', 'Debian') + when: ansible_distribution in ('Ubuntu') diff --git a/test/integration/roles/test_service/tasks/main.yml b/test/integration/roles/test_service/tasks/main.yml index 8b61d62143..de7c29a9cb 100644 --- a/test/integration/roles/test_service/tasks/main.yml +++ b/test/integration/roles/test_service/tasks/main.yml @@ -13,7 +13,7 @@ - include: 'sysv_setup.yml' when: ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and (ansible_distribution_version|version_compare('6', '>=') and ansible_distribution_version|version_compare('7', '<')) - include: 'systemd_setup.yml' - when: (ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and (ansible_distribution_version|version_compare('7', '>=') and ansible_distribution_version|version_compare('8', '<'))) or ansible_distribution == 'Fedora' or (ansible_distribution == 'Ubuntu' and ansible_distribution_version|version_compare('15.04', '>=')) + when: (ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and (ansible_distribution_version|version_compare('7', '>=') and ansible_distribution_version|version_compare('8', '<'))) or ansible_distribution == 'Fedora' or (ansible_distribution == 'Ubuntu' and ansible_distribution_version|version_compare('15.04', '>=')) or (ansible_distribution == 'Debian' and ansible_distribution_version|version_compare('8', '>=')) - include: 'upstart_setup.yml' when: ansible_distribution == 'Ubuntu' and ansible_distribution_version|version_compare('15.04', '<') diff --git a/test/integration/roles/test_service/tasks/systemd_setup.yml b/test/integration/roles/test_service/tasks/systemd_setup.yml index 4a3a81a4a6..d1428149d0 100644 --- a/test/integration/roles/test_service/tasks/systemd_setup.yml +++ b/test/integration/roles/test_service/tasks/systemd_setup.yml @@ -1,18 +1,18 @@ - name: install the systemd unit file - copy: src=ansible.systemd dest=/usr/lib/systemd/system/ansible_test.service + copy: src=ansible.systemd dest=/etc/systemd/system/ansible_test.service register: install_systemd_result - name: install a broken systemd unit file - file: src=ansible_test.service path=/usr/lib/systemd/system/ansible_test_broken.service state=link + file: src=ansible_test.service path=/etc/systemd/system/ansible_test_broken.service state=link register: install_broken_systemd_result - name: assert that the systemd unit file was installed assert: that: - - "install_systemd_result.dest == '/usr/lib/systemd/system/ansible_test.service'" + - "install_systemd_result.dest == '/etc/systemd/system/ansible_test.service'" - "install_systemd_result.state == 'file'" - "install_systemd_result.mode == '0644'" - "install_systemd_result.checksum == 'ca4b413fdf3cb2002f51893b9e42d2e449ec5afb'" - - "install_broken_systemd_result.dest == '/usr/lib/systemd/system/ansible_test_broken.service'" + - "install_broken_systemd_result.dest == '/etc/systemd/system/ansible_test_broken.service'" - "install_broken_systemd_result.state == 'link'" From 39e4caafb2367b258bb546cd21a4f804c559d21e Mon Sep 17 00:00:00 2001 From: Rene Moser Date: Sat, 30 Apr 2016 00:30:18 +0200 Subject: [PATCH 009/133] tests, apt_repository: disable Ubuntu 16.04 as there is no package yet --- test/integration/roles/test_apt_repository/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/roles/test_apt_repository/tasks/main.yml b/test/integration/roles/test_apt_repository/tasks/main.yml index 38ae8f0447..96c10e8c89 100644 --- a/test/integration/roles/test_apt_repository/tasks/main.yml +++ b/test/integration/roles/test_apt_repository/tasks/main.yml @@ -17,5 +17,5 @@ # along with Ansible. If not, see . - include: 'apt.yml' - when: ansible_distribution in ('Ubuntu') + when: ansible_distribution in ('Ubuntu') and ansible_distribution_version|version_compare('16.04', '<') From 5583027f99745c7b37575cfcad0b17153a0119a7 Mon Sep 17 00:00:00 2001 From: Rene Moser Date: Sun, 1 May 2016 00:52:17 +0200 Subject: [PATCH 010/133] tests, postgresql: add ubuntu 16.04 support --- .../roles/setup_postgresql_db/vars/Ubuntu-16.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test/integration/roles/setup_postgresql_db/vars/Ubuntu-16.yml diff --git a/test/integration/roles/setup_postgresql_db/vars/Ubuntu-16.yml b/test/integration/roles/setup_postgresql_db/vars/Ubuntu-16.yml new file mode 100644 index 0000000000..7b6092a12c --- /dev/null +++ b/test/integration/roles/setup_postgresql_db/vars/Ubuntu-16.yml @@ -0,0 +1,10 @@ +postgresql_service: "postgresql" + +postgresql_packages: + - "postgresql" + - "postgresql-common" + - "python-psycopg2" + +pg_hba_location: "/etc/postgresql/9.5/main/pg_hba.conf" +pg_dir: "/var/lib/postgresql/9.5/main" +pg_ver: 9.5 From fbfc24fb40507b38bf8fd77737bfaac4d34246ef Mon Sep 17 00:00:00 2001 From: Matt Hite Date: Tue, 3 May 2016 15:32:12 -0700 Subject: [PATCH 011/133] New inventory_ip_type option in gce inventory tool --- contrib/inventory/gce.ini | 8 ++++++ contrib/inventory/gce.py | 55 +++++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/contrib/inventory/gce.ini b/contrib/inventory/gce.ini index fd3325c79f..31f49704e6 100644 --- a/contrib/inventory/gce.ini +++ b/contrib/inventory/gce.ini @@ -45,3 +45,11 @@ gce_service_account_email_address = gce_service_account_pem_file_path = gce_project_id = +[inventory] +# The 'inventory_ip_type' parameter specifies whether 'ansible_ssh_host' should +# contain the instance internal or external address. Values may be either +# 'internal' or 'external'. If 'external' is specified but no external instance +# address exists, the internal address will be used. +# The INVENTORY_IP_TYPE environment variable will override this value. +inventory_ip_type = + diff --git a/contrib/inventory/gce.py b/contrib/inventory/gce.py index 690d845a02..2b402de740 100755 --- a/contrib/inventory/gce.py +++ b/contrib/inventory/gce.py @@ -69,7 +69,8 @@ Examples: $ contrib/inventory/gce.py --host my_instance Author: Eric Johnson -Version: 0.0.1 +Contributors: Matt Hite +Version: 0.0.2 ''' __requires__ = ['pycrypto>=2.6'] @@ -83,7 +84,7 @@ except ImportError: pass USER_AGENT_PRODUCT="Ansible-gce_inventory_plugin" -USER_AGENT_VERSION="v1" +USER_AGENT_VERSION="v2" import sys import os @@ -111,7 +112,11 @@ class GceInventory(object): def __init__(self): # Read settings and parse CLI arguments self.parse_cli_args() + self.config = self.get_config() self.driver = self.get_gce_driver() + self.ip_type = self.get_inventory_options() + if self.ip_type: + self.ip_type = self.ip_type.lower() # Just display data for specific host if self.args.host: @@ -125,9 +130,13 @@ class GceInventory(object): pretty=self.args.pretty)) sys.exit(0) - def get_gce_driver(self): - """Determine the GCE authorization settings and return a - libcloud driver. + def get_config(self): + """ + Populates a SafeConfigParser object with defaults and + attempts to read an .ini-style configuration from the filename + specified in GCE_INI_PATH. If the environment variable is + not present, the filename defaults to gce.ini in the current + working directory. """ gce_ini_default_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "gce.ini") @@ -142,14 +151,32 @@ class GceInventory(object): 'gce_service_account_pem_file_path': '', 'gce_project_id': '', 'libcloud_secrets': '', + 'inventory_ip_type': '', }) if 'gce' not in config.sections(): config.add_section('gce') - config.read(gce_ini_path) + if 'inventory' not in config.sections(): + config.add_section('inventory') + config.read(gce_ini_path) + return config + + def get_inventory_options(self): + """Determine inventory options. Environment variables always + take precedence over configuration files.""" + ip_type = self.config.get('inventory', 'inventory_ip_type') + # If the appropriate environment variables are set, they override + # other configuration + ip_type = os.environ.get('INVENTORY_IP_TYPE', ip_type) + return ip_type + + def get_gce_driver(self): + """Determine the GCE authorization settings and return a + libcloud driver. + """ # Attempt to get GCE params from a configuration file, if one # exists. - secrets_path = config.get('gce', 'libcloud_secrets') + secrets_path = self.config.get('gce', 'libcloud_secrets') secrets_found = False try: import secrets @@ -175,10 +202,10 @@ class GceInventory(object): pass if not secrets_found: args = [ - config.get('gce','gce_service_account_email_address'), - config.get('gce','gce_service_account_pem_file_path') + self.config.get('gce','gce_service_account_email_address'), + self.config.get('gce','gce_service_account_pem_file_path') ] - kwargs = {'project': config.get('gce', 'gce_project_id')} + kwargs = {'project': self.config.get('gce', 'gce_project_id')} # If the appropriate environment variables are set, they override # other configuration; process those into our args and kwargs. @@ -218,6 +245,12 @@ class GceInventory(object): md[entry['key']] = entry['value'] net = inst.extra['networkInterfaces'][0]['network'].split('/')[-1] + # default to exernal IP unless user has specified they prefer internal + if self.ip_type == 'internal': + ssh_host = inst.private_ips[0] + else: + ssh_host = inst.public_ips[0] if len(inst.public_ips) >= 1 else inst.private_ips[0] + return { 'gce_uuid': inst.uuid, 'gce_id': inst.id, @@ -233,7 +266,7 @@ class GceInventory(object): 'gce_metadata': md, 'gce_network': net, # Hosts don't have a public name, so we add an IP - 'ansible_ssh_host': inst.public_ips[0] if len(inst.public_ips) >= 1 else inst.private_ips[0] + 'ansible_ssh_host': ssh_host } def get_instance(self, instance_name): From 6373f2b0455ad023aef0f567d9c6469c12ba7f85 Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Wed, 4 May 2016 09:43:27 -0700 Subject: [PATCH 012/133] error message cleanup --- lib/ansible/plugins/connection/winrm.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py index 4c00e89d21..f5023d7efc 100644 --- a/lib/ansible/plugins/connection/winrm.py +++ b/lib/ansible/plugins/connection/winrm.py @@ -33,7 +33,6 @@ from ansible.errors import AnsibleError, AnsibleConnectionFailure try: import winrm from winrm import Response - from winrm.exceptions import WinRMTransportError from winrm.protocol import Protocol except ImportError: raise AnsibleError("winrm is not installed") @@ -122,7 +121,7 @@ class Connection(ConnectionBase): # warn for kwargs unsupported by the installed version of pywinrm for arg in unsupported_args: - display.warning("ansible_winrm_{0} unsupported by pywinrm (are you running the right pywinrm version?)".format(arg)) + display.warning("ansible_winrm_{0} unsupported by pywinrm (is an up-to-date version of pywinrm installed?)".format(arg)) # arg names we're going passing directly internal_kwarg_mask = set(['self', 'endpoint', 'transport', 'username', 'password']) @@ -147,9 +146,8 @@ class Connection(ConnectionBase): display.vvvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), host=self._winrm_host) try: protocol = Protocol(endpoint, transport=transport, **self._winrm_kwargs) - # send keepalive message to ensure we're awake - # TODO: is this necessary? - # protocol.send_message(xmltodict.unparse(rq)) + + # open the shell from connect so we know we're able to talk to the server if not self.shell_id: self.shell_id = protocol.open_shell(codepage=65001) # UTF-8 display.vvvvv('WINRM OPEN SHELL: %s' % self.shell_id, host=self._winrm_host) @@ -163,7 +161,7 @@ class Connection(ConnectionBase): if m: code = int(m.groups()[0]) if code == 401: - err_msg = 'the username/password specified for this server was incorrect' + err_msg = 'the specified credentials were rejected by the server' elif code == 411: return protocol errors.append(u'%s: %s' % (transport, err_msg)) @@ -282,7 +280,7 @@ class Connection(ConnectionBase): try: result.std_err = self.parse_clixml_stream(result.std_err) except: - # unsure if we're guaranteed a valid xml doc- keep original output just in case + # unsure if we're guaranteed a valid xml doc- use raw output in case of error pass return (result.status_code, result.std_out, result.std_err) @@ -294,7 +292,7 @@ class Connection(ConnectionBase): def parse_clixml_stream(self, clixml_doc, stream_name='Error'): clear_xml = clixml_doc.replace('#< CLIXML\r\n', '') doc = xmltodict.parse(clear_xml) - lines = [l.get('#text', '') for l in doc.get('Objs', {}).get('S', {}) if l.get('@S') == stream_name] + lines = [l.get('#text', '').replace('_x000D__x000A_', '') for l in doc.get('Objs', {}).get('S', {}) if l.get('@S') == stream_name] return '\r\n'.join(lines) # FUTURE: determine buffer size at runtime via remote winrm config? From 4f7a0925fda2d9ea9b0e9f41b21420055468622d Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 4 May 2016 12:17:42 -0700 Subject: [PATCH 013/133] Corrections to documentation formatting --- docsite/rst/developing_core.rst | 24 +++++++++++++++++++ docsite/rst/developing_modules.rst | 14 +++++------ .../rst/developing_program_flow_modules.rst | 6 ++--- docsite/rst/faq.rst | 2 +- docsite/rst/guide_azure.rst | 1 + 5 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 docsite/rst/developing_core.rst diff --git a/docsite/rst/developing_core.rst b/docsite/rst/developing_core.rst new file mode 100644 index 0000000000..c0067d74f8 --- /dev/null +++ b/docsite/rst/developing_core.rst @@ -0,0 +1,24 @@ +Developing the Ansible Core Engine +================================== + +Although many of the pieces of the Ansible Core Engine are plugins that can be +swapped out via playbook directives or configuration, there are still pieces +of the Engine that are not modular. The documents here give insight into how +those pieces work together. + +.. toctree:: + :maxdepth: 1 + + developing_program_flow_modules + +.. seealso:: + + :doc:`developing_api` + Learn about the Python API for task execution + :doc:`developing_plugins` + Learn about developing plugins + `Mailing List `_ + The development mailing list + `irc.freenode.net `_ + #ansible-devel IRC chat channel + diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index 89ca9b978e..d1a2cdf44c 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -48,8 +48,8 @@ the 'command' module could already be used to do this. Reading the modules that come with Ansible (linked above) is a great way to learn how to write modules. Keep in mind, though, that some modules in Ansible's source tree are internalisms, -so look at :ref:`service` or :ref:`yum`, and don't stare too close into things like :ref:`async_wrapper` or -you'll turn to stone. Nobody ever executes :ref:`async_wrapper` directly. +so look at :ref:`service` or :ref:`yum`, and don't stare too close into things like ``async_wrapper`` or +you'll turn to stone. Nobody ever executes ``async_wrapper`` directly. Ok, let's get going with an example. We'll use Python. For starters, save this as a file named :file:`timetest.py`:: @@ -538,11 +538,11 @@ When you look into the debug_dir you'll see a directory structure like this:: that are passed to the module, this is the file to do it in. * The :file:`ansible` directory contains code from - :module:`ansible.module_utils` that is used by the module. Ansible includes + :mod:`ansible.module_utils` that is used by the module. Ansible includes files for any :`module:`ansible.module_utils` imports in the module but not no files from any other module. So if your module uses - :module:`ansible.module_utils.url` Ansible will include it for you, but if - your module includes :module:`requests` then you'll have to make sure that + :mod:`ansible.module_utils.url` Ansible will include it for you, but if + your module includes :mod:`requests` then you'll have to make sure that the python requests library is installed on the system before running the module. You can modify files in this directory if you suspect that the module is having a problem in some of this boilerplate code rather than in @@ -566,7 +566,7 @@ module file and test that the real module works via :command:`ansible` or The wrapper provides one more subcommand, ``excommunicate``. This subcommand is very similar to ``execute`` in that it invokes the exploded module on the arguments in the :file:`args`. The way it does this is - different, however. ``excommunicate`` imports the :function:`main` + different, however. ``excommunicate`` imports the :func:`main` function from the module and then calls that. This makes excommunicate execute the module in the wrapper's process. This may be useful for running the module under some graphical debuggers but it is very different @@ -575,7 +575,7 @@ module file and test that the real module works via :command:`ansible` or with Ansible normally. Those are not bugs in the module; they're limitations of ``excommunicate``. Use at your own risk. -.. _module_paths +.. _module_paths: Module Paths ```````````` diff --git a/docsite/rst/developing_program_flow_modules.rst b/docsite/rst/developing_program_flow_modules.rst index c4f821231e..3bedeaf122 100644 --- a/docsite/rst/developing_program_flow_modules.rst +++ b/docsite/rst/developing_program_flow_modules.rst @@ -79,7 +79,7 @@ New-style powershell modules use the :ref:`module_replacer` framework for constructing modules. These modules get a library of powershell code embedded in them before being sent to the managed node. -.. _flow_josnargs_modules: +.. _flow_jsonargs_modules: JSONARGS ^^^^^^^^ @@ -325,7 +325,7 @@ string and substituted into the combined module file. In :ref:`ziploader`, the JSON-ified string is passed into the module via stdin. When a :class:`ansible.module_utils.basic.AnsibleModule` is instantiated, it parses this string and places the args into -:attribute:`AnsibleModule.params` where it can be accessed by the module's +:attr:`AnsibleModule.params` where it can be accessed by the module's other code. .. _flow_passing_module_constants: @@ -357,7 +357,7 @@ For now, :code:`ANSIBLE_VERSION` is also available at its old location inside of :ref:`ziploader` passes these as part of the JSON-ified argument string via stdin. When :class:`ansible.module_utils.basic.AnsibleModule` is instantiated, it parses this -string and places the constants into :attribute:`AnsibleModule.constants` +string and places the constants into :attr:`AnsibleModule.constants` where other code can access it. Unlike the ``ANSIBLE_VERSION``, where some efforts were made to keep the old diff --git a/docsite/rst/faq.rst b/docsite/rst/faq.rst index b0d1846dfc..cd0f191e78 100644 --- a/docsite/rst/faq.rst +++ b/docsite/rst/faq.rst @@ -329,7 +329,7 @@ be applied to single tasks only, once a playbook is completed. .. _interpolate_variables: When should I use {{ }}? Also, how to interpolate variables or dynamic variable names -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ A steadfast rule is 'always use {{ }} except when `when:`'. Conditionals are always run through Jinja2 as to resolve the expression, diff --git a/docsite/rst/guide_azure.rst b/docsite/rst/guide_azure.rst index c5c11a8830..de47a5b7de 100644 --- a/docsite/rst/guide_azure.rst +++ b/docsite/rst/guide_azure.rst @@ -332,6 +332,7 @@ A sample azure_rm.ini file is included along with the inventory script in contri file will contain the following: .. code-block:: ini + [azure] # Control which resource groups are included. By default all resources groups are included. # Set resource_groups to a comma separated list of resource groups names. From 56ba10365c3bbf7ad33dffbf33c96535d2347fb1 Mon Sep 17 00:00:00 2001 From: Robin Roth Date: Wed, 4 May 2016 21:32:08 +0200 Subject: [PATCH 014/133] better fix for arch version detection (#15705) * better fix for arch version detection fixes #15696 * be extra safe about tracebacks in facts.py * add comments to explain the setup * make allowempty more conservative, ignore file content * wrap function call in try/except * should never happen, but if it happens the bug should be distribtion=N/A and not a traceback --- lib/ansible/module_utils/facts.py | 38 ++++++++++++++++++------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index d847876f48..690403f810 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -604,6 +604,10 @@ class Distribution(object): This is unit tested. Please extend the tests to cover all distributions if you have them available. """ + # every distribution name mentioned here, must have one of + # - allowempty == True + # - be listed in SEARCH_STRING + # - have a function get_distribution_DISTNAME implemented OSDIST_LIST = ( {'path': '/etc/oracle-release', 'name': 'OracleLinux'}, {'path': '/etc/slackware-version', 'name': 'Slackware'}, @@ -687,12 +691,12 @@ class Distribution(object): if not os.path.exists(path): continue + # if allowempty is set, we only check for file existance but not content + if 'allowempty' in ddict and ddict['allowempty']: + self.facts['distribution'] = name + break if os.path.getsize(path) == 0: - if 'allowempty' in ddict and ddict['allowempty']: - self.facts['distribution'] = name - break - else: - continue + continue data = get_file_content(path) if name in self.SEARCH_STRING: @@ -707,13 +711,19 @@ class Distribution(object): break else: # call a dedicated function for parsing the file content - distfunc = getattr(self, 'get_distribution_' + name) - parsed = distfunc(name, data, path) - if parsed is None or parsed: - # distfunc return False if parsing failed - # break only if parsing was succesful - # otherwise continue with other distributions - break + try: + distfunc = getattr(self, 'get_distribution_' + name) + parsed = distfunc(name, data, path) + if parsed is None or parsed: + # distfunc return False if parsing failed + # break only if parsing was succesful + # otherwise continue with other distributions + break + except AttributeError: + # this should never happen, but if it does fail quitely and not with a traceback + pass + + # to debug multiple matching release files, one can use: # self.facts['distribution_debug'].append({path + ' ' + name: @@ -780,10 +790,6 @@ class Distribution(object): if release: self.facts['distribution_release'] = release.groups()[0] - def get_distribution_Archlinux(self, name, data, path): - self.facts['distribution'] = 'Archlinux' - self.facts['distribution_version'] = data - def get_distribution_Alpine(self, name, data, path): self.facts['distribution'] = 'Alpine' self.facts['distribution_version'] = data From bc81c76f8654b77fa18c79b71d6bc46919b9e04f Mon Sep 17 00:00:00 2001 From: Andrew Taumoefolau Date: Thu, 5 May 2016 22:02:13 +1000 Subject: [PATCH 015/133] Apply inventory host restrictions by host name rather than UUID. Issue #15633 observes that a meta: inventory_refresh task causes the playbook to exit. An inventory refresh flushes all caches and rebuilds all host objects, assigning new UUIDs to each. These new host UUIDs currently fail to match those on host objects stored for restrictions in the inventory, causing the playbook to exit for having no hosts to run further tasks against. This changeset attempts to address this issue by storing host restrictions by name, and comparing inventory host names against these names when applying restrictions in get_hosts. --- lib/ansible/inventory/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index 4797b6977f..abb8b0b082 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -204,7 +204,7 @@ class Inventory(object): # exclude hosts mentioned in any restriction (ex: failed hosts) if self._restriction is not None: - hosts = [ h for h in hosts if h in self._restriction ] + hosts = [ h for h in hosts if h.name in self._restriction ] seen = set() HOSTS_PATTERNS_CACHE[pattern_hash] = [x for x in hosts if x not in seen and not seen.add(x)] @@ -600,7 +600,7 @@ class Inventory(object): return elif not isinstance(restriction, list): restriction = [ restriction ] - self._restriction = restriction + self._restriction = [ h.name for h in restriction ] def subset(self, subset_pattern): """ From be87cd8c265391bbc8c482e7aaaa7bc8477a6355 Mon Sep 17 00:00:00 2001 From: camradal Date: Thu, 5 May 2016 06:48:54 -0700 Subject: [PATCH 016/133] Fix logging into vCloud Director and expose verify_certs argument (#15533) --- lib/ansible/module_utils/vca.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/ansible/module_utils/vca.py b/lib/ansible/module_utils/vca.py index 92e51183d3..e895d28a7d 100644 --- a/lib/ansible/module_utils/vca.py +++ b/lib/ansible/module_utils/vca.py @@ -46,7 +46,8 @@ def vca_argument_spec(): api_version=dict(default=DEFAULT_VERSION), service_type=dict(default=DEFAULT_SERVICE_TYPE, choices=SERVICE_MAP.keys()), vdc_name=dict(), - gateway_name=dict(default='gateway') + gateway_name=dict(default='gateway'), + verify_certs=dict(type='bool', default=True) ) class VcaAnsibleModule(AnsibleModule): @@ -130,7 +131,11 @@ class VcaAnsibleModule(AnsibleModule): service_type = self.params['service_type'] password = self.params['password'] - if not self.vca.login(password=password): + login_org = None + if service_type == 'vcd': + login_org = self.params['org'] + + if not self.vca.login(password=password, org=login_org): self.fail('Login to VCA failed', response=self.vca.response.content) try: From fb7940fc50cc9f5dd5e794e6c8149f89084cfe05 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 May 2016 11:14:11 -0400 Subject: [PATCH 017/133] check that variable first before using string methods to check for magic interpreter var --- lib/ansible/executor/task_executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 0af2e2b35c..2a475c2a10 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -595,14 +595,14 @@ class TaskExecutor: # since we're delegating, we don't want to use interpreter values # which would have been set for the original target host for i in variables.keys(): - if i.startswith('ansible_') and i.endswith('_interpreter'): + if isinstance(i, string_types) and i.startswith('ansible_') and i.endswith('_interpreter'): del variables[i] # now replace the interpreter values with those that may have come # from the delegated-to host delegated_vars = variables.get('ansible_delegated_vars', dict()).get(self._task.delegate_to, dict()) if isinstance(delegated_vars, dict): for i in delegated_vars: - if i.startswith("ansible_") and i.endswith("_interpreter"): + if isinstance(i, string_types) and i.startswith("ansible_") and i.endswith("_interpreter"): variables[i] = delegated_vars[i] conn_type = self._play_context.connection From 27a1ae4732ef0aad9dd31fa80431e39259ed3c05 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 May 2016 11:32:45 -0400 Subject: [PATCH 018/133] added ability to also subset make tests --- test/utils/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index 20ddaaf613..33b8c4779e 100755 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -15,7 +15,7 @@ else export C_NAME="testAbull_$$_$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)" docker pull ansible/ansible:${TARGET} docker run -d --volume="${PWD}:/root/ansible:Z" --name "${C_NAME}" ${TARGET_OPTIONS:=''} ansible/ansible:${TARGET} > /tmp/cid_${TARGET} - docker exec -ti $(cat /tmp/cid_${TARGET}) /bin/sh -c "export TEST_FLAGS='${TEST_FLAGS:-''}'; cd /root/ansible; . hacking/env-setup; (cd test/integration; LC_ALL=en_US.utf-8 make)" + docker exec -ti $(cat /tmp/cid_${TARGET}) /bin/sh -c "export TEST_FLAGS='${TEST_FLAGS:-''}'; cd /root/ansible; . hacking/env-setup; (cd test/integration; LC_ALL=en_US.utf-8 make ${MAKE_TARGET:-})" docker kill $(cat /tmp/cid_${TARGET}) if [ "X${TESTS_KEEP_CONTAINER:-''}" = "X" ]; then From 2af8e3b9d8b7e5c14ea61abeeccf705dc00a050d Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 May 2016 15:59:47 -0400 Subject: [PATCH 019/133] fix default for removing images --- test/utils/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index 33b8c4779e..f58b108386 100755 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -18,7 +18,7 @@ else docker exec -ti $(cat /tmp/cid_${TARGET}) /bin/sh -c "export TEST_FLAGS='${TEST_FLAGS:-''}'; cd /root/ansible; . hacking/env-setup; (cd test/integration; LC_ALL=en_US.utf-8 make ${MAKE_TARGET:-})" docker kill $(cat /tmp/cid_${TARGET}) - if [ "X${TESTS_KEEP_CONTAINER:-''}" = "X" ]; then + if [ "X${TESTS_KEEP_CONTAINER:-""}" = "X" ]; then docker rm -vf "${C_NAME}" fi fi From b7c874f81a1ceccc7e35c9bf4a5efff6ab1ff48a Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 May 2016 16:55:18 -0400 Subject: [PATCH 020/133] enforce required 'required' in docs --- hacking/module_formatter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 07a9c540c1..7afba19da9 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -289,6 +289,8 @@ def process_module(module, options, env, template, outputname, module_map, alias del doc['options'][k]['version_added'] if not 'description' in doc['options'][k]: raise AnsibleError("Missing required description for option %s in %s " % (k, module)) + if not 'requried' in doc['options'][k]: + raise AnsibleError("Missing required 'required' for option %s in %s " % (k, module)) if not isinstance(doc['options'][k]['description'],list): doc['options'][k]['description'] = [doc['options'][k]['description']] From 3a6ca0b4a6153fc42e7fe4c32159d4844e6d52fb Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 May 2016 17:02:39 -0400 Subject: [PATCH 021/133] made ansible-doc complain on missing 'requried' --- lib/ansible/cli/doc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index 65b4327aea..27c3eea9d3 100644 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -219,7 +219,9 @@ class DocCLI(CLI): opt = doc['options'][o] desc = CLI.tty_ify(" ".join(opt['description'])) - required = opt.get('required', False) + required = opt.get('required') + if required is None: + raise("Missing required field 'Required'") if not isinstance(required, bool): raise("Incorrect value for 'Required', a boolean is needed.: %s" % required) if required: From 478674cc57788e6b8a47de3560abf3f54e5113d1 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 May 2016 17:05:46 -0400 Subject: [PATCH 022/133] typo fix --- hacking/module_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 7afba19da9..72245e533e 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -289,7 +289,7 @@ def process_module(module, options, env, template, outputname, module_map, alias del doc['options'][k]['version_added'] if not 'description' in doc['options'][k]: raise AnsibleError("Missing required description for option %s in %s " % (k, module)) - if not 'requried' in doc['options'][k]: + if not 'required' in doc['options'][k]: raise AnsibleError("Missing required 'required' for option %s in %s " % (k, module)) if not isinstance(doc['options'][k]['description'],list): doc['options'][k]['description'] = [doc['options'][k]['description']] From 85868e07a9a4641c845ad1be3d036e716ff89bad Mon Sep 17 00:00:00 2001 From: Andrew Taumoefolau Date: Wed, 4 May 2016 15:14:28 +1000 Subject: [PATCH 023/133] Don't assume a task with non-dict loop results has been skipped. This changeset addresses the issue reported here: ansible/ansible-modules-core#1765 The yum module (at least) includes its task results as strings, rather than dicts, and the code this changeset replaces assumed that in that instance the task was skipped. The updated behaviour assumes that the task has been skipped only if: * results exist, and * all results are dicts that include a truthy skipped value --- lib/ansible/executor/task_result.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/ansible/executor/task_result.py b/lib/ansible/executor/task_result.py index da9ab2a11a..db73f1ccb9 100644 --- a/lib/ansible/executor/task_result.py +++ b/lib/ansible/executor/task_result.py @@ -41,11 +41,8 @@ class TaskResult: def is_skipped(self): if 'results' in self._result and self._task.loop: - flag = True - for res in self._result.get('results', []): - if isinstance(res, dict): - flag &= res.get('skipped', False) - return flag + results = self._result['results'] + return results and all(isinstance(res, dict) and res.get('skipped', False) for res in results) else: return self._result.get('skipped', False) From 133395db30110175b02325a23ff5156077f71458 Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Thu, 5 May 2016 15:26:40 -0700 Subject: [PATCH 024/133] add jimi-c's unit test for squashed skip results, tweaked is_skipped() logic to pass --- lib/ansible/executor/task_result.py | 11 ++++++++--- test/units/executor/test_task_result.py | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/ansible/executor/task_result.py b/lib/ansible/executor/task_result.py index db73f1ccb9..457a094b73 100644 --- a/lib/ansible/executor/task_result.py +++ b/lib/ansible/executor/task_result.py @@ -40,11 +40,16 @@ class TaskResult: return self._check_key('changed') def is_skipped(self): + # loop results if 'results' in self._result and self._task.loop: results = self._result['results'] - return results and all(isinstance(res, dict) and res.get('skipped', False) for res in results) - else: - return self._result.get('skipped', False) + # Loop tasks are only considered skipped if all items were skipped. + # some squashed results (eg, yum) are not dicts and can't be skipped individually + if results and all(isinstance(res, dict) and res.get('skipped', False) for res in results): + return True + + # regular tasks and squashed non-dict results + return self._result.get('skipped', False) def is_failed(self): if 'failed_when_result' in self._result or \ diff --git a/test/units/executor/test_task_result.py b/test/units/executor/test_task_result.py index a0af67edbd..c83d7b4489 100644 --- a/test/units/executor/test_task_result.py +++ b/test/units/executor/test_task_result.py @@ -85,6 +85,15 @@ class TestTaskResult(unittest.TestCase): tr = TaskResult(mock_host, mock_task, dict(results=[dict(skipped=True), dict(skipped=True), dict(skipped=True)])) self.assertTrue(tr.is_skipped()) + # test with multiple squashed results (list of strings) + # first with the main result having skipped=False + mock_task.loop = 'foo' + tr = TaskResult(mock_host, mock_task, dict(results=["a", "b", "c"], skipped=False)) + self.assertFalse(tr.is_skipped()) + # then with the main result having skipped=True + tr = TaskResult(mock_host, mock_task, dict(results=["a", "b", "c"], skipped=True)) + self.assertTrue(tr.is_skipped()) + def test_task_result_is_unreachable(self): mock_host = MagicMock() mock_task = MagicMock() From 7708948d7d59bda1fb620b8ba3c05a56abb1c18c Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Thu, 5 May 2016 17:05:36 -0700 Subject: [PATCH 025/133] bump submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 9db1233521..9be870bb2a 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 9db123352168e81f8c81e95225a64663d2724d5b +Subproject commit 9be870bb2a5f277ac7ba68e87243b74dcc0ab1ac diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 675d778b50..2e09202aae 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 675d778b50257a72290a68f4f644a4f630e85ce4 +Subproject commit 2e09202aae9e8f7934e4b05a8305f7c1352a61e6 From 5c7ad654db25e6e2d930ca9375b9b44bd9e2564a Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 5 May 2016 20:33:02 -0700 Subject: [PATCH 026/133] Add some more tests for item squashing --- test/units/executor/test_task_executor.py | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/units/executor/test_task_executor.py b/test/units/executor/test_task_executor.py index 3bca43b702..8f2c1ed8cc 100644 --- a/test/units/executor/test_task_executor.py +++ b/test/units/executor/test_task_executor.py @@ -260,6 +260,44 @@ class TestTaskExecutor(unittest.TestCase): new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, items) + job_vars = dict(pkg_mgr='yum') + items = [{'package': 'foo'}, {'package': 'bar'}] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ items["package"] }}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + self.assertEqual(new_items, items) + + # Could do something like this to recover from bad deps in a package + job_vars = dict(pkg_mgr='yum', packages=['a', 'b']) + items = [ 'absent', 'latest' ] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + self.assertEqual(new_items, items) + + # + # These are presently not optimized but could be in the future. + # Expected output if they were optimized is given as a comment + # Please move these to a different section if they are optimized + # + + job_vars = dict(pkg_mgr='yum') + items = [['a', 'b'], ['foo', 'bar']] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ packages[item] }}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + self.assertEqual(new_items, items) + #self.assertEqual(new_items, ['a', 'b', 'foo', 'bar']) + + ### Enable this when we've fixed https://github.com/ansible/ansible/issues/15649 + #job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "c": "baz" }) + #items = ['a', 'b', 'c'] + #mock_task.action = 'yum' + #mock_task.args = {'name': '{{ packages[item] }}'} + #new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + #self.assertEqual(new_items, items) + #self.assertEqual(new_items, ['foo', 'bar', 'baz']) + def test_task_executor_execute(self): fake_loader = DictDataLoader({}) From 21ac95402f77ccbf2c19f03f37334d08ac24630e Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Thu, 5 May 2016 23:45:06 -0400 Subject: [PATCH 027/133] do not erroneously set gathered_facts=True In `lib/ansible/executor/play_iterator.py`, ansible sets a host's `_gathered_facts` property to `True` without checking to see if there are any tasks to be executed. In the event that the entire play is skipped, `_gathered_facts` will be `True` even though the `setup` module was never run. This patch modifies the logic to only set `_gathered_facts` to `True` when there are tasks to execute. Closes #15744. --- lib/ansible/executor/play_iterator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ansible/executor/play_iterator.py b/lib/ansible/executor/play_iterator.py index 3ab2e289ac..b5fc8bf5c3 100644 --- a/lib/ansible/executor/play_iterator.py +++ b/lib/ansible/executor/play_iterator.py @@ -295,10 +295,10 @@ class PlayIterator: setup_block = self._blocks[0] if setup_block.has_tasks() and len(setup_block.block) > 0: task = setup_block.block[0] - if not peek: - # mark the host as having gathered facts, because we're - # returning the setup task to be executed - host.set_gathered_facts(True) + if not peek: + # mark the host as having gathered facts, because we're + # returning the setup task to be executed + host.set_gathered_facts(True) else: # This is the second trip through ITERATING_SETUP, so we clear # the flag and move onto the next block in the list while setting From 09c90f7f2f7b193ebdb9c51573b37678f35db192 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Fri, 29 Apr 2016 08:32:31 -0400 Subject: [PATCH 028/133] Fixing bugs in strategies * Don't filter hosts remaining based on their failed state. Instead rely on the PlayIterator to return None/ITERATING_COMPLETE when the host is failed. * In the free strategy, make sure we wait outside the host loop for all pending results to be processed. * Use the internal _set_failed_state() instead of manually setting things when a failed child state is hit Fixes #15623 --- lib/ansible/executor/play_iterator.py | 11 +++++------ lib/ansible/plugins/strategy/free.py | 5 ++++- lib/ansible/plugins/strategy/linear.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/ansible/executor/play_iterator.py b/lib/ansible/executor/play_iterator.py index 3ab2e289ac..9ecda9395c 100644 --- a/lib/ansible/executor/play_iterator.py +++ b/lib/ansible/executor/play_iterator.py @@ -326,8 +326,7 @@ class PlayIterator: if self._check_failed_state(state.tasks_child_state): # failed child state, so clear it and move into the rescue portion state.tasks_child_state = None - state.fail_state |= self.FAILED_TASKS - state.run_state = self.ITERATING_RESCUE + self._set_failed_state(state) else: # get the next task recursively if task is None or state.tasks_child_state.run_state == self.ITERATING_COMPLETE: @@ -365,8 +364,7 @@ class PlayIterator: (state.rescue_child_state, task) = self._get_next_task_from_state(state.rescue_child_state, host=host, peek=peek) if self._check_failed_state(state.rescue_child_state): state.rescue_child_state = None - state.fail_state |= self.FAILED_RESCUE - state.run_state = self.ITERATING_ALWAYS + self._set_failed_state(state) else: if task is None or state.rescue_child_state.run_state == self.ITERATING_COMPLETE: state.rescue_child_state = None @@ -396,8 +394,7 @@ class PlayIterator: (state.always_child_state, task) = self._get_next_task_from_state(state.always_child_state, host=host, peek=peek) if self._check_failed_state(state.always_child_state): state.always_child_state = None - state.fail_state |= self.FAILED_ALWAYS - state.run_state = self.ITERATING_COMPLETE + self._set_failed_state(state) else: if task is None or state.always_child_state.run_state == self.ITERATING_COMPLETE: state.always_child_state = None @@ -466,7 +463,9 @@ class PlayIterator: def mark_host_failed(self, host): s = self.get_host_state(host) + display.debug("marking host %s failed, current state: %s" % (host, s)) s = self._set_failed_state(s) + display.debug("^ failed state is now: %s" % s) self._host_states[host.name] = s def get_failed_hosts(self): diff --git a/lib/ansible/plugins/strategy/free.py b/lib/ansible/plugins/strategy/free.py index b64b5df53b..fb0ae7dfcc 100644 --- a/lib/ansible/plugins/strategy/free.py +++ b/lib/ansible/plugins/strategy/free.py @@ -58,7 +58,7 @@ class StrategyModule(StrategyBase): work_to_do = True while work_to_do and not self._tqm._terminated: - hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts and not iterator.is_failed(host)] + hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts] if len(hosts_left) == 0: self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') result = False @@ -191,6 +191,9 @@ class StrategyModule(StrategyBase): # pause briefly so we don't spin lock time.sleep(0.001) + # collect all the final results + results = self._wait_on_pending_results(iterator) + # run the base class run() method, which executes the cleanup function # and runs any outstanding handlers which have been triggered return super(StrategyModule, self).run(iterator, play_context, result) diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py index aa0f67e1da..9ab302e0da 100644 --- a/lib/ansible/plugins/strategy/linear.py +++ b/lib/ansible/plugins/strategy/linear.py @@ -163,7 +163,7 @@ class StrategyModule(StrategyBase): try: display.debug("getting the remaining hosts for this loop") - hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts and not iterator.is_failed(host)] + hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts] display.debug("done getting the remaining hosts for this loop") # queue up this task for each host in the inventory From 589f6d25bb7b2cf15d519a877db81e3b2bbf886e Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Fri, 6 May 2016 14:53:43 +0100 Subject: [PATCH 029/133] Updating VCA module documentation (#14368) --- lib/ansible/utils/module_docs_fragments/vca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/utils/module_docs_fragments/vca.py b/lib/ansible/utils/module_docs_fragments/vca.py index 88cb1b4184..a67a64316b 100644 --- a/lib/ansible/utils/module_docs_fragments/vca.py +++ b/lib/ansible/utils/module_docs_fragments/vca.py @@ -34,7 +34,7 @@ options: aliases: ['pass', 'pwd'] org: description: - - The org to login to for creating vapp, mostly set when the service_type is vdc. + - The org to login to for creating vapp. This option is required when the C(service_type) is I(vdc). required: false default: None instance_id: From a274ef9adc1f5cd60903f53bade9c06a3100737c Mon Sep 17 00:00:00 2001 From: Miguel Cabrerizo Date: Fri, 6 May 2016 15:57:16 +0200 Subject: [PATCH 030/133] Update intro_dynamic_inventory.rst (#15750) When using Cobbler with Ansible a cobbler.ini file is needed by the /etc/ansible/cobbler.py script, otherwise Python errors like ConfigParser.NoSectionError: No section: 'cobbler' are shown. Maybe this could be added to the documentation. Thanks and cheers! --- docsite/rst/intro_dynamic_inventory.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index 07223de653..5c4731d360 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -34,6 +34,28 @@ To tie Ansible's inventory to Cobbler (optional), copy `this script Date: Fri, 6 May 2016 08:13:51 -0700 Subject: [PATCH 031/133] Update submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 9be870bb2a..1f5cf669dd 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 9be870bb2a5f277ac7ba68e87243b74dcc0ab1ac +Subproject commit 1f5cf669ddec2ea46cd5f335f56f0d493403c99a diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 2e09202aae..d1b16cd007 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 2e09202aae9e8f7934e4b05a8305f7c1352a61e6 +Subproject commit d1b16cd0075b4428df2f7036beade8c18e154cde From fca5ba153e9258d6a9a28c418d8339d507eee81c Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Fri, 6 May 2016 09:49:33 -0700 Subject: [PATCH 032/133] bump extras submodule ref --- lib/ansible/modules/extras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index d1b16cd007..431591c2b4 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit d1b16cd0075b4428df2f7036beade8c18e154cde +Subproject commit 431591c2b45f28ac4033e04953d8ecfc360ba575 From 48ef69982b2c3d27486f0e0f8bfb5bb00298800b Mon Sep 17 00:00:00 2001 From: Scott Radvan Date: Sun, 8 May 2016 00:16:47 +1200 Subject: [PATCH 033/133] s/2015/2016 --- RELEASES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASES.txt b/RELEASES.txt index 766ff1a227..377bc82ab8 100644 --- a/RELEASES.txt +++ b/RELEASES.txt @@ -10,7 +10,7 @@ Released ++++++++ 2.1.0 "The Song Remains the Same" in progress -2.0.2 "Over the Hills and Far Away" 04-19-2015 +2.0.2 "Over the Hills and Far Away" 04-19-2016 2.0.1 "Over the Hills and Far Away" 02-24-2016 2.0.0 "Over the Hills and Far Away" 01-12-2016 1.9.6 "Dancing In the Streets" 04-15-2016 From 78808fc4ccea06dab1ed20a643c64f56fa89527f Mon Sep 17 00:00:00 2001 From: Pomin Wu Date: Mon, 9 May 2016 21:59:26 +0800 Subject: [PATCH 034/133] Fixed `ansible_os_family` variable on OS X (#15768) (#15773) Fixed `ansible_os_family` variable on OS X --- lib/ansible/module_utils/facts.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index 690403f810..d0884bced4 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -179,7 +179,7 @@ class Facts(object): # about those first. if load_on_init: self.get_platform_facts() - self.facts.update(Distribution().populate()) + self.facts.update(Distribution(module).populate()) self.get_cmdline() self.get_public_ssh_host_keys() self.get_selinux_facts() @@ -647,17 +647,19 @@ class Distribution(object): FreeBSD = 'FreeBSD', HPUX = 'HP-UX', openSUSE_Leap = 'Suse' ) - def __init__(self): + def __init__(self, module): self.system = platform.system() self.facts = {} + self.module = module def populate(self): if self.system == 'Linux': self.get_distribution_facts() + elif self.system == 'Darwin': + self.get_distribution_facts() return self.facts def get_distribution_facts(self): - # The platform module provides information about the running # system/distribution. Use this as a baseline and fix buggy systems # afterwards From 72ca3b2b5ba0e0256a21d6f4c459426854fdf09d Mon Sep 17 00:00:00 2001 From: Scott Radvan Date: Tue, 10 May 2016 02:12:24 +1200 Subject: [PATCH 035/133] add very minor punctuation fixes (#15763) --- docs/man/man1/ansible-playbook.1.asciidoc.in | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/man/man1/ansible-playbook.1.asciidoc.in b/docs/man/man1/ansible-playbook.1.asciidoc.in index 47e68d3141..c9170503ac 100644 --- a/docs/man/man1/ansible-playbook.1.asciidoc.in +++ b/docs/man/man1/ansible-playbook.1.asciidoc.in @@ -46,7 +46,7 @@ Ask for privilege escalation password. *-k*, *--ask-pass*:: Prompt for the connection password, if it is needed for the transport used. -For example, using ssh and not having a key-based authentication with ssh-agent. +For example, using ssh and not having a key-based authentication with ssh-agent. *--ask-su-pass*:: @@ -96,7 +96,7 @@ Level of parallelism. 'NUM' is specified as an integer, the default is 5. *-h*, *--help*:: -Show help page and exit +Show help page and exit. *-i* 'PATH', *--inventory=*'PATH':: @@ -128,7 +128,7 @@ environment variable. *--private-key=*'PRIVATE_KEY_FILE':: -Use this file to authenticate the connection +Use this file to authenticate the connection. *--start-at-task=*'START_AT':: @@ -140,11 +140,11 @@ One-step-at-a-time: confirm each task before running. *-S*, --su*:: -Run operations with su (deprecated, use become) +Run operations with su (deprecated, use become). *-R SU-USER*, *--su-user=*'SU_USER':: -run operations with su as this user (default=root) (deprecated, use become) +run operations with su as this user (default=root) (deprecated, use become). *-s*, *--sudo*:: @@ -178,7 +178,7 @@ Only run plays and tasks whose tags do not match these values. *--syntax-check*:: -Look for syntax errors in the playbook, but don't run anything +Look for syntax errors in the playbook, but don't run anything. *-t*, 'TAGS', *--tags=*'TAGS':: @@ -227,7 +227,7 @@ EXIT STATUS ENVIRONMENT ----------- -The following environment variables may be specified. +The following environment variables may be specified: ANSIBLE_INVENTORY -- Override the default ansible inventory file From 30e59998126e8255365009e36400e4c7e3abffe8 Mon Sep 17 00:00:00 2001 From: Robin Roth Date: Mon, 9 May 2016 16:55:28 +0200 Subject: [PATCH 036/133] Fix distribution_facts missing on BSD (#15780) The previous fix in #15773 only solved MacOSX, but left other BSDs broken fixes #15768 --- lib/ansible/module_utils/facts.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index d0884bced4..fa23d059b0 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -653,10 +653,7 @@ class Distribution(object): self.module = module def populate(self): - if self.system == 'Linux': - self.get_distribution_facts() - elif self.system == 'Darwin': - self.get_distribution_facts() + self.get_distribution_facts() return self.facts def get_distribution_facts(self): @@ -678,7 +675,7 @@ class Distribution(object): cleanedname = self.system.replace('-','') distfunc = getattr(self, 'get_distribution_'+cleanedname) distfunc() - else: + elif self.system == 'Linux': # try to find out which linux distribution this is dist = platform.dist() self.facts['distribution'] = dist[0].capitalize() or 'NA' From 6309ca4883eee6a6e88826e8b2fb456e89427006 Mon Sep 17 00:00:00 2001 From: Jose Andres Alvarez Loaiciga Date: Mon, 9 May 2016 09:05:43 -0600 Subject: [PATCH 037/133] Fix typo in `ansible-playbook` --- docsite/rst/intro_dynamic_inventory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index 5c4731d360..d851bd190a 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -133,7 +133,7 @@ If you use boto profiles to manage multiple AWS accounts, you can pass ``--profi aws_access_key_id = aws_secret_access_key = -You can then run ``ec2.py --profile prod`` to get the inventory for the prod account, this option is not supported by ``anisble-playbook`` though. +You can then run ``ec2.py --profile prod`` to get the inventory for the prod account, this option is not supported by ``ansible-playbook`` though. But you can use the ``AWS_PROFILE`` variable - e.g. ``AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml`` Since each region requires its own API call, if you are only using a small set of regions, feel free to edit ``ec2.ini`` and list only the regions you are interested in. There are other config options in ``ec2.ini`` including cache control, and destination variables. From 409bfe4d0f1effa307b83898818ac9e78fb9443f Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 9 May 2016 08:14:08 -0700 Subject: [PATCH 038/133] Strip leading and trailing whitespace for json arg types --- lib/ansible/module_utils/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index d8ab7a6b15..2190b3462f 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1400,7 +1400,7 @@ class AnsibleModule(object): # Return a jsonified string. Sometimes the controller turns a json # string into a dict/list so transform it back into json here if isinstance(value, (unicode, bytes)): - return value + return value.strip() else: if isinstance(value (list, tuple, dict)): return json.dumps(value) From 781de837621770378919bf5f3f256414ce522694 Mon Sep 17 00:00:00 2001 From: jctanner Date: Mon, 9 May 2016 14:24:39 -0400 Subject: [PATCH 039/133] When walking through module directories, always follow symlinks. (#15784) Fixes #15783 --- lib/ansible/plugins/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index e8558fef0a..19b217319e 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -136,7 +136,7 @@ class PluginLoader: def _all_directories(self, dir): results = [] results.append(dir) - for root, subdirs, files in os.walk(dir): + for root, subdirs, files in os.walk(dir, followlinks=True): if '__init__.py' in files: for x in subdirs: results.append(os.path.join(root,x)) From 724e692f5420ccabbe741c48bac7c838dbe1a28b Mon Sep 17 00:00:00 2001 From: Kamjar Gerami Date: Mon, 9 May 2016 22:48:46 +0200 Subject: [PATCH 040/133] fixes-#15685-tools-that-paginate-show-spurious-less-output: less --version outputs to standard out not to standard error so this changes the redirect from 2> to > (#15720) fixes-#15685-tools-that-paginate-show-spurious-less-output: Updated redirect to include stderr as well as stdout to not show any errors on screen --- lib/ansible/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 00c29524e7..ff8bf75490 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -476,7 +476,7 @@ class CLI(object): display.display(text) else: self.pager_pipe(text, os.environ['PAGER']) - elif subprocess.call('(less --version) 2> /dev/null', shell = True) == 0: + elif subprocess.call('(less --version) &> /dev/null', shell = True) == 0: self.pager_pipe(text, 'less') else: display.display(text) From 30decd5e0ac27c6fd68a530f334f72deed9698e5 Mon Sep 17 00:00:00 2001 From: Darren Birkett Date: Mon, 9 May 2016 23:27:18 +0100 Subject: [PATCH 041/133] Fix a couple of typos in guide_rax.rst (#15795) --- docsite/rst/guide_rax.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/guide_rax.rst b/docsite/rst/guide_rax.rst index 814f9bcef8..b2ad6c0c0c 100644 --- a/docsite/rst/guide_rax.rst +++ b/docsite/rst/guide_rax.rst @@ -156,7 +156,7 @@ to the next section. Host Inventory `````````````` -Once your nodes are spun up, you'll probably want to talk to them again. The best way to handle his is to use the "rax" inventory plugin, which dynamically queries Rackspace Cloud and tells Ansible what nodes you have to manage. You might want to use this even if you are spinning up Ansible via other tools, including the Rackspace Cloud user interface. The inventory plugin can be used to group resources by metadata, region, OS, etc. Utilizing metadata is highly recommended in "rax" and can provide an easy way to sort between host groups and roles. If you don't want to use the ``rax.py`` dynamic inventory script, you could also still choose to manually manage your INI inventory file, though this is less recommended. +Once your nodes are spun up, you'll probably want to talk to them again. The best way to handle this is to use the "rax" inventory plugin, which dynamically queries Rackspace Cloud and tells Ansible what nodes you have to manage. You might want to use this even if you are spinning up cloud instances via other tools, including the Rackspace Cloud user interface. The inventory plugin can be used to group resources by metadata, region, OS, etc. Utilizing metadata is highly recommended in "rax" and can provide an easy way to sort between host groups and roles. If you don't want to use the ``rax.py`` dynamic inventory script, you could also still choose to manually manage your INI inventory file, though this is less recommended. In Ansible it is quite possible to use multiple dynamic inventory plugins along with INI file data. Just put them in a common directory and be sure the scripts are chmod +x, and the INI-based ones are not. From 254cf9ea6821a54086a59dba5c4fe443c28a2679 Mon Sep 17 00:00:00 2001 From: Robin Roth Date: Tue, 10 May 2016 08:09:28 +0200 Subject: [PATCH 042/133] reduce async timeout reduce from 3 sec to 0.1 sec; the 3 sec sleep was about half of the total unittest time on my development machine... --- test/units/executor/test_task_executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/units/executor/test_task_executor.py b/test/units/executor/test_task_executor.py index 8f2c1ed8cc..1838111f54 100644 --- a/test/units/executor/test_task_executor.py +++ b/test/units/executor/test_task_executor.py @@ -372,8 +372,8 @@ class TestTaskExecutor(unittest.TestCase): mock_host = MagicMock() mock_task = MagicMock() - mock_task.async = 3 - mock_task.poll = 1 + mock_task.async = 0.1 + mock_task.poll = 0.05 mock_play_context = MagicMock() From 395f85e1bff6fe899490cca5ad87ffff2b17cddf Mon Sep 17 00:00:00 2001 From: Victor Bocharsky Date: Tue, 10 May 2016 10:48:59 +0300 Subject: [PATCH 043/133] Highlight SSH protocol types --- docsite/rst/intro_inventory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/intro_inventory.rst b/docsite/rst/intro_inventory.rst index b63800704e..d64eb130eb 100644 --- a/docsite/rst/intro_inventory.rst +++ b/docsite/rst/intro_inventory.rst @@ -203,7 +203,7 @@ As alluded to above, setting the following variables controls how ansible intera Host connection: ansible_connection - Connection type to the host. This can be the name of any of ansible's connection plugins. SSH protocol types are smart, ssh or paramiko. The default is smart. Non-SSH based types are described in the next section. + Connection type to the host. This can be the name of any of ansible's connection plugins. SSH protocol types are ``smart``, ``ssh`` or ``paramiko``. The default is smart. Non-SSH based types are described in the next section. .. include:: ansible_ssh_changes_note.rst From 7ccb08cc7ec008ed99a87fec7de3be81fa3fac10 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 10 May 2016 07:08:47 -0700 Subject: [PATCH 044/133] Switch to a different url for testing SNI right now. (#15798) --- test/integration/roles/test_uri/tasks/main.yml | 17 +++++++++++------ test/integration/roles/test_uri/vars/main.yml | 5 +++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/test/integration/roles/test_uri/tasks/main.yml b/test/integration/roles/test_uri/tasks/main.yml index a9db1dedb0..6b84a963eb 100644 --- a/test/integration/roles/test_uri/tasks/main.yml +++ b/test/integration/roles/test_uri/tasks/main.yml @@ -211,9 +211,13 @@ set_fact: is_ubuntu_precise: "{{ ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'precise' }}" +# These tests are just side effects of how the site is hosted. It's not +# specifically a test site. So the tests may break due to the hosting +# changing. Eventually we need to standup a webserver with SNI as part of the +# test run. - name: Test that SNI succeeds on python versions that have SNI uri: - url: 'https://sni.velox.ch' + url: "{{ SNI_URI }}" return_content: true when: ansible_python.has_sslcontext register: result @@ -221,13 +225,13 @@ - name: Assert SNI verification succeeds on new python assert: that: - - result|success - - '"Great! Your client" in result.content' + - result|success + - "\"

If You Can Read This, You're SNIing

\" in result.content" when: ansible_python.has_sslcontext - name: Verify SNI verification fails on old python without urllib3 contrib uri: - url: 'https://sni.velox.ch' + url: '{{ SNI_URI }}' ignore_errors: true when: not ansible_python.has_sslcontext register: result @@ -253,7 +257,7 @@ - name: Verify SNI verificaiton succeeds on old python with urllib3 contrib uri: - url: 'https://sni.velox.ch' + url: '{{ SNI_URI }}' return_content: true when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool register: result @@ -262,7 +266,8 @@ assert: that: - result|success - - '"Great! Your client" in result.content' + #- '"Great! Your client" in result.content' + - "\"

If You Can Read This, You're SNIing

\" in result.content" when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool - name: Uninstall ndg-httpsclient and urllib3 diff --git a/test/integration/roles/test_uri/vars/main.yml b/test/integration/roles/test_uri/vars/main.yml index b404a14a28..b819276f65 100644 --- a/test/integration/roles/test_uri/vars/main.yml +++ b/test/integration/roles/test_uri/vars/main.yml @@ -7,3 +7,8 @@ uri_os_packages: - python-pyasn1 - python-openssl - python-urllib3 + +# Needs to be a url to a site that is hosted using SNI. +# Eventually we should make this a test server that we stand up as part of the test run. +#SNI_URI: 'https://sni.velox.ch' +SNI_URI: "https://www.mnot.net/blog/2014/05/09/if_you_can_read_this_youre_sniing" From 9e88fa21f528a8449c3e5f19a20712c929293716 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 10 May 2016 07:13:57 -0700 Subject: [PATCH 045/133] Update submodule refs --- lib/ansible/modules/extras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 431591c2b4..816673dd6f 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 431591c2b45f28ac4033e04953d8ecfc360ba575 +Subproject commit 816673dd6f5df0f9f27753c1c2cadf22ea70befb From ddd9f9225578e812c6a6f75577a19ebcd98be5f0 Mon Sep 17 00:00:00 2001 From: Matthew Stoltenberg Date: Tue, 10 May 2016 08:23:10 -0600 Subject: [PATCH 046/133] add repr for hostvars (#15793) * allows passing full hostvars to a module --- lib/ansible/vars/hostvars.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/ansible/vars/hostvars.py b/lib/ansible/vars/hostvars.py index 2370b8f8a8..c4010447b7 100644 --- a/lib/ansible/vars/hostvars.py +++ b/lib/ansible/vars/hostvars.py @@ -99,3 +99,9 @@ class HostVars(collections.Mapping): def __len__(self): return len(self._inventory.get_hosts(ignore_limits_and_restrictions=True)) + def __repr__(self): + out = {} + for host in self._inventory.get_hosts(ignore_limits_and_restrictions=True): + name = host.name + out[name] = self.get(name) + return repr(out) From c730af5dc22a47c13c0339afe79e3ad87a6a845c Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 10 May 2016 08:13:48 -0700 Subject: [PATCH 047/133] Remove reload from arg related tests. Changes to how ziploader passes args mean we don't need reload anymore. (#15782) --- .../module_utils/basic/test_exit_json.py | 8 ++-- test/units/module_utils/basic/test_log.py | 44 +++++++++---------- .../module_utils/basic/test_run_command.py | 2 +- .../module_utils/basic/test_safe_eval.py | 8 +--- test/units/module_utils/test_basic.py | 40 +++++++---------- 5 files changed, 44 insertions(+), 58 deletions(-) diff --git a/test/units/module_utils/basic/test_exit_json.py b/test/units/module_utils/basic/test_exit_json.py index 73a97ed687..2cc6239845 100644 --- a/test/units/module_utils/basic/test_exit_json.py +++ b/test/units/module_utils/basic/test_exit_json.py @@ -28,8 +28,6 @@ from ansible.compat.tests import unittest from units.mock.procenv import swap_stdin_and_argv, swap_stdout from ansible.module_utils import basic -from ansible.module_utils.basic import heuristic_log_sanitize -from ansible.module_utils.basic import return_values, remove_values empty_invocation = {u'module_args': {}} @@ -45,7 +43,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase): self.stdout_swap_ctx = swap_stdout() self.fake_stream = self.stdout_swap_ctx.__enter__() - reload(basic) + basic._ANSIBLE_ARGS = None self.module = basic.AnsibleModule(argument_spec=dict()) def tearDown(self): @@ -126,8 +124,8 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase): params = json.dumps(params) with swap_stdin_and_argv(stdin_data=params): - reload(basic) with swap_stdout(): + basic._ANSIBLE_ARGS = None module = basic.AnsibleModule( argument_spec = dict( username=dict(), @@ -148,8 +146,8 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase): params = dict(ANSIBLE_MODULE_ARGS=args, ANSIBLE_MODULE_CONSTANTS={}) params = json.dumps(params) with swap_stdin_and_argv(stdin_data=params): - reload(basic) with swap_stdout(): + basic._ANSIBLE_ARGS = None module = basic.AnsibleModule( argument_spec = dict( username=dict(), diff --git a/test/units/module_utils/basic/test_log.py b/test/units/module_utils/basic/test_log.py index f9a00bb55e..6e47e46c00 100644 --- a/test/units/module_utils/basic/test_log.py +++ b/test/units/module_utils/basic/test_log.py @@ -28,7 +28,7 @@ from ansible.compat.tests import unittest from ansible.compat.tests.mock import patch, MagicMock from units.mock.procenv import swap_stdin_and_argv -from ansible.module_utils import basic +import ansible.module_utils.basic try: @@ -49,20 +49,20 @@ class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase): self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() - reload(basic) - self.am = basic.AnsibleModule( + ansible.module_utils.basic._ANSIBLE_ARGS = None + self.am = ansible.module_utils.basic.AnsibleModule( argument_spec = dict(), ) - self.has_journal = basic.has_journal + self.has_journal = ansible.module_utils.basic.has_journal if self.has_journal: # Systems with journal can still test syslog - basic.has_journal = False + ansible.module_utils.basic.has_journal = False def tearDown(self): # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap.__exit__(None, None, None) - basic.has_journal = self.has_journal + ansible.module_utils.basic.has_journal = self.has_journal def test_smoketest_syslog(self): # These talk to the live daemons on the system. Need to do this to @@ -86,8 +86,8 @@ class TestAnsibleModuleJournaldSmokeTest(unittest.TestCase): self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() - reload(basic) - self.am = basic.AnsibleModule( + ansible.module_utils.basic._ANSIBLE_ARGS = None + self.am = ansible.module_utils.basic.AnsibleModule( argument_spec = dict(), ) @@ -95,7 +95,7 @@ class TestAnsibleModuleJournaldSmokeTest(unittest.TestCase): # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap.__exit__(None, None, None) - @unittest.skipUnless(basic.has_journal, 'python systemd bindings not installed') + @unittest.skipUnless(ansible.module_utils.basic.has_journal, 'python systemd bindings not installed') def test_smoketest_journal(self): # These talk to the live daemons on the system. Need to do this to # show that what we send doesn't cause an issue once it gets to the @@ -134,19 +134,19 @@ class TestAnsibleModuleLogSyslog(unittest.TestCase): self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() - reload(basic) - self.am = basic.AnsibleModule( + ansible.module_utils.basic._ANSIBLE_ARGS = None + self.am = ansible.module_utils.basic.AnsibleModule( argument_spec = dict(), ) - self.has_journal = basic.has_journal + self.has_journal = ansible.module_utils.basic.has_journal if self.has_journal: # Systems with journal can still test syslog - basic.has_journal = False + ansible.module_utils.basic.has_journal = False def tearDown(self): # teardown/reset - basic.has_journal = self.has_journal + ansible.module_utils.basic.has_journal = self.has_journal # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap.__exit__(None, None, None) @@ -195,13 +195,13 @@ class TestAnsibleModuleLogJournal(unittest.TestCase): self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() - reload(basic) - self.am = basic.AnsibleModule( + ansible.module_utils.basic._ANSIBLE_ARGS = None + self.am = ansible.module_utils.basic.AnsibleModule( argument_spec = dict(), ) - self.has_journal = basic.has_journal - basic.has_journal = True + self.has_journal = ansible.module_utils.basic.has_journal + ansible.module_utils.basic.has_journal = True self.module_patcher = None @@ -210,20 +210,20 @@ class TestAnsibleModuleLogJournal(unittest.TestCase): self.module_patcher = patch.dict('sys.modules', {'systemd': MagicMock(), 'systemd.journal': MagicMock()}) self.module_patcher.start() try: - reload(basic) + reload(ansible.module_utils.basic) except NameError: - self._fake_out_reload(basic) + self._fake_out_reload(ansible.module_utils.basic) def tearDown(self): # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap.__exit__(None, None, None) # teardown/reset - basic.has_journal = self.has_journal + ansible.module_utils.basic.has_journal = self.has_journal if self.module_patcher: self.module_patcher.stop() - reload(basic) + reload(ansible.module_utils.basic) @patch('systemd.journal.send') def test_no_log(self, mock_func): diff --git a/test/units/module_utils/basic/test_run_command.py b/test/units/module_utils/basic/test_run_command.py index aaf9b0e134..a78825d46b 100644 --- a/test/units/module_utils/basic/test_run_command.py +++ b/test/units/module_utils/basic/test_run_command.py @@ -67,7 +67,7 @@ class TestAnsibleModuleRunCommand(unittest.TestCase): self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() - reload(basic) + basic._ANSIBLE_ARGS = None self.module = AnsibleModule(argument_spec=dict()) self.module.fail_json = MagicMock(side_effect=SystemExit) diff --git a/test/units/module_utils/basic/test_safe_eval.py b/test/units/module_utils/basic/test_safe_eval.py index e38ed9b46e..cc2aaf8a53 100644 --- a/test/units/module_utils/basic/test_safe_eval.py +++ b/test/units/module_utils/basic/test_safe_eval.py @@ -26,12 +26,6 @@ import json from ansible.compat.tests import unittest from units.mock.procenv import swap_stdin_and_argv -try: - from importlib import reload -except: - # Py2 has reload as a builtin - pass - class TestAnsibleModuleExitJson(unittest.TestCase): def test_module_utils_basic_safe_eval(self): @@ -40,7 +34,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase): args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) with swap_stdin_and_argv(stdin_data=args): - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec=dict(), ) diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index 662d3fe37c..42825ff304 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -31,12 +31,6 @@ try: except ImportError: import __builtin__ as builtins -try: - from importlib import reload -except: - # Py2 has reload as a builtin - pass - from units.mock.procenv import swap_stdin_and_argv from ansible.compat.tests import unittest @@ -297,7 +291,7 @@ class TestModuleUtilsBasic(unittest.TestCase): args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo": "hello"}, ANSIBLE_MODULE_CONSTANTS={})) with swap_stdin_and_argv(stdin_data=args): - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = arg_spec, mutually_exclusive = mut_ex, @@ -314,7 +308,7 @@ class TestModuleUtilsBasic(unittest.TestCase): args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) with swap_stdin_and_argv(stdin_data=args): - reload(basic) + basic._ANSIBLE_ARGS = None self.assertRaises( SystemExit, basic.AnsibleModule, @@ -361,7 +355,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_load_file_common_arguments(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -410,7 +404,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_selinux_mls_enabled(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -430,7 +424,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_selinux_initial_context(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -444,7 +438,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_selinux_enabled(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -476,7 +470,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_selinux_default_context(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -512,7 +506,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_selinux_context(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -554,7 +548,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_is_special_selinux_path(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={"SELINUX_SPECIAL_FS": "nfs,nfsd,foos"})) @@ -599,7 +593,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_to_filesystem_str(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -624,7 +618,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_find_mount_point(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -648,7 +642,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_set_context_if_different(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -693,7 +687,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_set_owner_if_different(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -732,7 +726,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_set_group_if_different(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -771,7 +765,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module_set_mode_if_different(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -859,7 +853,7 @@ class TestModuleUtilsBasic(unittest.TestCase): ): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), @@ -1037,7 +1031,7 @@ class TestModuleUtilsBasic(unittest.TestCase): def test_module_utils_basic_ansible_module__symbolic_mode_to_octal(self): from ansible.module_utils import basic - reload(basic) + basic._ANSIBLE_ARGS = None am = basic.AnsibleModule( argument_spec = dict(), From 47d05e9b58cc083f08f017fbadba9db38c29fdf0 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 10 May 2016 12:06:21 -0400 Subject: [PATCH 048/133] fix bad assignment, method modifies by ref already fixes #15694 --- lib/ansible/plugins/strategy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index 00af818841..25b2791577 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -349,7 +349,7 @@ class StrategyBase: # be a host that is not really in inventory at all if task.delegate_to is not None and task.delegate_facts: task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, host=host, task=task) - task_vars = self.add_tqm_variables(task_vars, play=iterator._play) + self.add_tqm_variables(task_vars, play=iterator._play) loop_var = 'item' if task.loop_control: loop_var = task.loop_control.loop_var or 'item' From dae1d4dbb7acf3c6ad8a11032d105186ccaa3ec2 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 10 May 2016 09:33:39 -0700 Subject: [PATCH 049/133] Add changelog entry for ec2_customer_gateway (in 2.2) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1faf3fb85..791b5db55d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Ansible Changes By Release ## 2.2 TBD - ACTIVE DEVELOPMENT +####New Modules: +- aws + * ec2_customer_gateway + ## 2.1 "The Song Remains the Same" - ACTIVE DEVELOPMENT ###Major Changes: From f576082949e1eab5d4fcdc46ee15b7fd31c82d75 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 10 May 2016 15:54:46 -0400 Subject: [PATCH 050/133] restore old jsonfile behaviour on key expiration fixes #14456, now it won't expire keys in middle of a play when they were 'valid' at 'gather time'. --- lib/ansible/plugins/cache/jsonfile.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/ansible/plugins/cache/jsonfile.py b/lib/ansible/plugins/cache/jsonfile.py index f0ec635ebe..a99f5a3e31 100644 --- a/lib/ansible/plugins/cache/jsonfile.py +++ b/lib/ansible/plugins/cache/jsonfile.py @@ -62,13 +62,16 @@ class CacheModule(BaseCacheModule): return None def get(self, key): - - if self.has_expired(key) or key == "": - raise KeyError + """ This checks the in memory cache first as the fact was not expired at 'gather time' + and it would be problematic if the key did expire after some long running tasks and + user gets 'undefined' error in the same play """ if key in self._cache: return self._cache.get(key) + if self.has_expired(key) or key == "": + raise KeyError + cachefile = "%s/%s" % (self._cache_dir, key) try: with codecs.open(cachefile, 'r', encoding='utf-8') as f: From a685fa55430d632719c40fe314156dae1136e380 Mon Sep 17 00:00:00 2001 From: Tim Rupp Date: Tue, 10 May 2016 13:07:01 -0700 Subject: [PATCH 051/133] Add port argument for bigsuds (#15434) This patch adds the port argument as a valid parameter to the f5_spec. This argument is supported in bigsuds version 1.0.4 and greater, so this patch uses the __version__ variable of the bigsuds module to determine when the port value should be honored by the module. --- lib/ansible/module_utils/f5.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/ansible/module_utils/f5.py b/lib/ansible/module_utils/f5.py index ba336377e7..78ebdd077c 100644 --- a/lib/ansible/module_utils/f5.py +++ b/lib/ansible/module_utils/f5.py @@ -43,6 +43,7 @@ def f5_argument_spec(): user=dict(type='str', required=True), password=dict(type='str', aliases=['pass', 'pwd'], required=True, no_log=True), validate_certs = dict(default='yes', type='bool'), + server_port = dict(type='int', default=443, required=False), state = dict(type='str', default='present', choices=['present', 'absent']), partition = dict(type='str', default='Common') ) @@ -57,12 +58,16 @@ def f5_parse_arguments(module): if not hasattr(ssl, 'SSLContext'): module.fail_json(msg='bigsuds does not support verifying certificates with python < 2.7.9. Either update python or set validate_certs=False on the task') - return (module.params['server'],module.params['user'],module.params['password'],module.params['state'],module.params['partition'],module.params['validate_certs']) + return (module.params['server'],module.params['user'],module.params['password'],module.params['state'],module.params['partition'],module.params['validate_certs'],module.params['server_port']) -def bigip_api(bigip, user, password, validate_certs): +def bigip_api(bigip, user, password, validate_certs, port=443): try: - # bigsuds >= 1.0.3 - api = bigsuds.BIGIP(hostname=bigip, username=user, password=password, verify=validate_certs) + if bigsuds.__version__ >= '1.0.4': + api = bigsuds.BIGIP(hostname=bigip, username=user, password=password, verify=validate_certs, port=port) + elif bigsuds.__version__ == '1.0.3': + api = bigsuds.BIGIP(hostname=bigip, username=user, password=password, verify=validate_certs) + else: + api = bigsuds.BIGIP(hostname=bigip, username=user, password=password) except TypeError: # bigsuds < 1.0.3, no verify param if validate_certs: @@ -92,5 +97,3 @@ def fq_list_names(partition,list_names): if list_names is None: return None return map(lambda x: fq_name(partition,x),list_names) - - From e33b97ed1fbe66ce686c132555b40b10f6c37b98 Mon Sep 17 00:00:00 2001 From: Devon Harvey Date: Tue, 10 May 2016 13:19:52 -0700 Subject: [PATCH 052/133] Fix typo in intro_dynamic_inventory.rst (#15810) --- docsite/rst/intro_dynamic_inventory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index 5c4731d360..d851bd190a 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -133,7 +133,7 @@ If you use boto profiles to manage multiple AWS accounts, you can pass ``--profi aws_access_key_id = aws_secret_access_key = -You can then run ``ec2.py --profile prod`` to get the inventory for the prod account, this option is not supported by ``anisble-playbook`` though. +You can then run ``ec2.py --profile prod`` to get the inventory for the prod account, this option is not supported by ``ansible-playbook`` though. But you can use the ``AWS_PROFILE`` variable - e.g. ``AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml`` Since each region requires its own API call, if you are only using a small set of regions, feel free to edit ``ec2.ini`` and list only the regions you are interested in. There are other config options in ``ec2.ini`` including cache control, and destination variables. From 4561be00af27222ed80893183372a944428245b2 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 10 May 2016 17:16:05 -0400 Subject: [PATCH 053/133] simplified inventory error messaging --- lib/ansible/inventory/dir.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/ansible/inventory/dir.py b/lib/ansible/inventory/dir.py index 29a0c754b8..5a3d25fe5e 100644 --- a/lib/ansible/inventory/dir.py +++ b/lib/ansible/inventory/dir.py @@ -52,6 +52,7 @@ def get_file_parser(hostsfile, groups, loader): except: pass + #FIXME: make this 'plugin loop' # script if loader.is_executable(hostsfile): @@ -59,9 +60,9 @@ def get_file_parser(hostsfile, groups, loader): parser = InventoryScript(loader=loader, groups=groups, filename=hostsfile) processed = True except Exception as e: - myerr.append("The file %s is marked as executable, but failed to execute correctly. " % hostsfile + \ - "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile) myerr.append(str(e)) + elif shebang_present: + myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. Perhaps you want to correct this with `chmod +x %s`?" % (hostsfile, hostsfile)) # YAML/JSON if not processed and os.path.splitext(hostsfile)[-1] in C.YAML_FILENAME_EXTENSIONS: @@ -69,11 +70,7 @@ def get_file_parser(hostsfile, groups, loader): parser = InventoryYAMLParser(loader=loader, groups=groups, filename=hostsfile) processed = True except Exception as e: - if shebang_present and not loader.is_executable(hostsfile): - myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \ - "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile) - else: - myerr.append(str(e)) + myerr.append(str(e)) # ini if not processed: @@ -81,11 +78,7 @@ def get_file_parser(hostsfile, groups, loader): parser = InventoryINIParser(loader=loader, groups=groups, filename=hostsfile) processed = True except Exception as e: - if shebang_present and not loader.is_executable(hostsfile): - myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \ - "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile) - else: - myerr.append(str(e)) + myerr.append(str(e)) if not processed and myerr: raise AnsibleError( '\n'.join(myerr) ) From accf40d8a854aa42ca17d6a01958f0ca5be23668 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 10 May 2016 22:43:07 -0500 Subject: [PATCH 054/133] Use httptester docker image for http tests (#15811) * Use httptester docker image for http tests * When not running with an httptester linked container, use public test sites --- test/integration/destructive.yml | 1 + test/integration/non_destructive.yml | 1 - .../prepare_http_tests/defaults/main.yml | 3 + .../roles/prepare_http_tests/tasks/main.yml | 35 ++++++++ .../prepare_http_tests/vars/httptester.yml | 4 + .../roles/test_get_url/meta/main.yml | 3 +- .../roles/test_get_url/tasks/main.yml | 89 ++++++++----------- .../roles/test_lookups/tasks/main.yml | 2 +- test/integration/roles/test_uri/meta/main.yml | 1 + .../integration/roles/test_uri/tasks/main.yml | 48 ++++------ test/utils/run_tests.sh | 8 +- 11 files changed, 111 insertions(+), 84 deletions(-) create mode 100644 test/integration/roles/prepare_http_tests/defaults/main.yml create mode 100644 test/integration/roles/prepare_http_tests/tasks/main.yml create mode 100644 test/integration/roles/prepare_http_tests/vars/httptester.yml diff --git a/test/integration/destructive.yml b/test/integration/destructive.yml index b63a315b90..f3400909de 100644 --- a/test/integration/destructive.yml +++ b/test/integration/destructive.yml @@ -21,4 +21,5 @@ - { role: test_zypper, tags: test_zypper} - { role: test_zypper_repository, tags: test_zypper_repository} - { role: test_uri, tags: test_uri } + - { role: test_get_url, tags: test_get_url } - { role: test_apache2_module, tags: test_apache2_module } diff --git a/test/integration/non_destructive.yml b/test/integration/non_destructive.yml index 369b09d16a..59910755d4 100644 --- a/test/integration/non_destructive.yml +++ b/test/integration/non_destructive.yml @@ -38,7 +38,6 @@ - { role: test_command_shell, tags: test_command_shell } - { role: test_script, tags: test_script } - { role: test_authorized_key, tags: test_authorized_key } - - { role: test_get_url, tags: test_get_url } - { role: test_embedded_module, tags: test_embedded_module } - { role: test_add_host, tags: test_add_host } - { role: test_binary, tags: test_binary } diff --git a/test/integration/roles/prepare_http_tests/defaults/main.yml b/test/integration/roles/prepare_http_tests/defaults/main.yml new file mode 100644 index 0000000000..c41faa2bd3 --- /dev/null +++ b/test/integration/roles/prepare_http_tests/defaults/main.yml @@ -0,0 +1,3 @@ +badssl_host: wrong.host.badssl.com +httpbin_host: httpbin.org +sni_host: sni.velox.ch diff --git a/test/integration/roles/prepare_http_tests/tasks/main.yml b/test/integration/roles/prepare_http_tests/tasks/main.yml new file mode 100644 index 0000000000..c3678aad79 --- /dev/null +++ b/test/integration/roles/prepare_http_tests/tasks/main.yml @@ -0,0 +1,35 @@ +# The docker --link functionality gives us an ENV var we can key off of to see if we have access to +# the httptester container +- set_fact: + has_httptester: "{{ lookup('env', 'ANSIBLE.HTTP.TESTS_PORT_80_TCP_ADDR') != '' }}" + +# If we are running with access to a httptester container, grab it's cacert and install it +- block: + # Override hostname defaults with httptester linked names + - include_vars: httptester.yml + + - name: RedHat - Enable the dynamic CA configuration feature + command: update-ca-trust force-enable + when: ansible_os_family == 'RedHat' + + - name: RedHat - Retrieve test cacert + get_url: + url: "http://ansible.http.tests/cacert.pem" + dest: "/etc/pki/ca-trust/source/anchors/ansible.pem" + when: ansible_os_family == 'RedHat' + + - name: Debian - Retrieve test cacert + get_url: + url: "http://ansible.http.tests/cacert.pem" + dest: "/usr/local/share/ca-certificates/ansible.crt" + when: ansible_os_family == 'Debian' + + - name: Redhat - Update ca trust + command: update-ca-trust extract + when: ansible_os_family == 'RedHat' + + - name: Debian - Update ca certificates + command: update-ca-certificates + when: ansible_os_family == 'Debian' + + when: has_httptester|bool diff --git a/test/integration/roles/prepare_http_tests/vars/httptester.yml b/test/integration/roles/prepare_http_tests/vars/httptester.yml new file mode 100644 index 0000000000..e5f46db630 --- /dev/null +++ b/test/integration/roles/prepare_http_tests/vars/httptester.yml @@ -0,0 +1,4 @@ +# these are fake hostnames provided by docker link for the httptester container +badssl_host: fail.ansible.http.tests +httpbin_host: ansible.http.tests +sni_host: sni1.ansible.http.tests diff --git a/test/integration/roles/test_get_url/meta/main.yml b/test/integration/roles/test_get_url/meta/main.yml index 1050c23ce3..b5f2416aed 100644 --- a/test/integration/roles/test_get_url/meta/main.yml +++ b/test/integration/roles/test_get_url/meta/main.yml @@ -1,3 +1,4 @@ -dependencies: +dependencies: - prepare_tests + - prepare_http_tests diff --git a/test/integration/roles/test_get_url/tasks/main.yml b/test/integration/roles/test_get_url/tasks/main.yml index 45ad1ab938..0bb0bfd854 100644 --- a/test/integration/roles/test_get_url/tasks/main.yml +++ b/test/integration/roles/test_get_url/tasks/main.yml @@ -66,27 +66,21 @@ - result.failed - name: test https fetch - get_url: url="https://raw.githubusercontent.com/ansible/ansible/devel/README.md" dest={{output_dir}}/get_url.txt force=yes + get_url: url="https://{{ httpbin_host }}/get" dest={{output_dir}}/get_url.txt force=yes register: result - name: assert the get_url call was successful assert: that: - - result.changed + - result.changed - '"OK" in result.msg' - name: test https fetch to a site with mismatched hostname and certificate get_url: - url: "https://www.kennethreitz.org/" + url: "https://{{ badssl_host }}/" dest: "{{ output_dir }}/shouldnotexist.html" ignore_errors: True register: result - # kennethreitz having trouble staying up. Eventually need to install our own - # certs & web server to test this... also need to install and test it with - # a proxy so the complications are inevitable - until: "'read operation timed out' not in result.msg" - retries: 30 - delay: 10 - stat: path: "{{ output_dir }}/shouldnotexist.html" @@ -101,16 +95,13 @@ - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no get_url: - url: "https://www.kennethreitz.org/" - dest: "{{ output_dir }}/kreitz.html" + url: "https://{{ badssl_host }}/" + dest: "{{ output_dir }}/get_url_no_validate.html" validate_certs: no register: result - until: "'read operation timed out' not in result.msg" - retries: 30 - delay: 10 - stat: - path: "{{ output_dir }}/kreitz.html" + path: "{{ output_dir }}/get_url_no_validate.html" register: stat_result - name: Assert that the file was downloaded @@ -119,48 +110,44 @@ - "result.changed == true" - "stat_result.stat.exists == true" -# At the moment, AWS can't make an https request to velox.ch... connection -# timed out. So we'll use a different test until/unless the problem is resolved -## SNI Tests -## SNI is only built into the stdlib from python-2.7.9 onwards -#- name: Test that SNI works -# get_url: -# # A test site that returns a page with information on what SNI information -# # the client sent. A failure would have the string: did not send a TLS server name indication extension -# url: 'https://foo.sni.velox.ch/' -# dest: "{{ output_dir }}/sni.html" -# register: get_url_result -# ignore_errors: True -# -#- command: "grep 'sent the following TLS server name indication extension' {{ output_dir}}/sni.html" -# register: data_result -# when: "{{ python_has_ssl_context }}" -# -#- debug: var=get_url_result -#- name: Assert that SNI works with this python version -# assert: -# that: -# - 'data_result.rc == 0' -# - '"failed" not in get_url_result' -# when: "{{ python_has_ssl_context }}" -# -## If the client doesn't support SNI then get_url should have failed with a certificate mismatch -#- name: Assert that hostname verification failed because SNI is not supported on this version of python -# assert: -# that: -# - 'get_url_result["failed"]' -# when: "{{ not python_has_ssl_context }}" +# SNI Tests +# SNI is only built into the stdlib from python-2.7.9 onwards +- name: Test that SNI works + get_url: + url: 'https://{{ sni_host }}/' + dest: "{{ output_dir }}/sni.html" + register: get_url_result + ignore_errors: True + +- command: "grep '{{ sni_host }}' {{ output_dir}}/sni.html" + register: data_result + when: "{{ python_has_ssl_context }}" + +- debug: var=get_url_result +- name: Assert that SNI works with this python version + assert: + that: + - 'data_result.rc == 0' + - '"failed" not in get_url_result' + when: "{{ python_has_ssl_context }}" + +# If the client doesn't support SNI then get_url should have failed with a certificate mismatch +- name: Assert that hostname verification failed because SNI is not supported on this version of python + assert: + that: + - 'get_url_result["failed"]' + when: "{{ not python_has_ssl_context }}" # These tests are just side effects of how the site is hosted. It's not # specifically a test site. So the tests may break due to the hosting changing - name: Test that SNI works get_url: - url: 'https://www.mnot.net/blog/2014/05/09/if_you_can_read_this_youre_sniing' + url: 'https://{{ sni_host }}/' dest: "{{ output_dir }}/sni.html" register: get_url_result ignore_errors: True -- command: "grep '

If You Can Read This, You.re SNIing

' {{ output_dir}}/sni.html" +- command: "grep '{{ sni_host }}' {{ output_dir}}/sni.html" register: data_result when: "{{ python_has_ssl_context }}" @@ -182,12 +169,12 @@ - name: Test get_url with redirect get_url: - url: 'http://httpbin.org/redirect/6' + url: 'http://{{ httpbin_host }}/redirect/6' dest: "{{ output_dir }}/redirect.json" - name: Test that setting file modes work get_url: - url: 'http://httpbin.org/' + url: 'http://{{ httpbin_host }}/' dest: '{{ output_dir }}/test' mode: '0707' register: result @@ -204,7 +191,7 @@ - name: Test that setting file modes on an already downlaoded file work get_url: - url: 'http://httpbin.org/' + url: 'http://{{ httpbin_host }}/' dest: '{{ output_dir }}/test' mode: '0070' register: result diff --git a/test/integration/roles/test_lookups/tasks/main.yml b/test/integration/roles/test_lookups/tasks/main.yml index 5b179690f1..72fa11be79 100644 --- a/test/integration/roles/test_lookups/tasks/main.yml +++ b/test/integration/roles/test_lookups/tasks/main.yml @@ -84,7 +84,7 @@ # ENV LOOKUP - name: get first environment var name - shell: env | head -n1 | cut -d\= -f1 + shell: env | fgrep -v '.' | head -n1 | cut -d\= -f1 register: known_var_name - name: get first environment var value diff --git a/test/integration/roles/test_uri/meta/main.yml b/test/integration/roles/test_uri/meta/main.yml index 07faa21776..a5f3f70736 100644 --- a/test/integration/roles/test_uri/meta/main.yml +++ b/test/integration/roles/test_uri/meta/main.yml @@ -1,2 +1,3 @@ dependencies: - prepare_tests + - prepare_http_tests diff --git a/test/integration/roles/test_uri/tasks/main.yml b/test/integration/roles/test_uri/tasks/main.yml index 6b84a963eb..7f2ee952a4 100644 --- a/test/integration/roles/test_uri/tasks/main.yml +++ b/test/integration/roles/test_uri/tasks/main.yml @@ -94,16 +94,10 @@ - name: test https fetch to a site with mismatched hostname and certificate uri: - url: "https://www.kennethreitz.org/" + url: "https://{{ badssl_host }}/" dest: "{{ output_dir }}/shouldnotexist.html" ignore_errors: True register: result - # kennethreitz having trouble staying up. Eventually need to install our own - # certs & web server to test this... also need to install and test it with - # a proxy so the complications are inevitable - until: "'read operation timed out' not in result.msg" - retries: 30 - delay: 10 - stat: path: "{{ output_dir }}/shouldnotexist.html" @@ -123,13 +117,10 @@ - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no uri: - url: "https://www.kennethreitz.org/" + url: "https://{{ badssl_host }}/" dest: "{{ output_dir }}/kreitz.html" validate_certs: no register: result - until: "'read operation timed out' not in result.msg" - retries: 30 - delay: 10 - stat: path: "{{ output_dir }}/kreitz.html" @@ -143,7 +134,7 @@ - name: test redirect without follow_redirects uri: - url: 'http://httpbin.org/redirect/2' + url: 'http://{{ httpbin_host }}/redirect/2' follow_redirects: 'none' status_code: 302 register: result @@ -151,21 +142,21 @@ - name: Assert location header assert: that: - - 'result.location|default("") == "http://httpbin.org/relative-redirect/1"' + - 'result.location|default("") == "http://{{ httpbin_host }}/relative-redirect/1"' - name: Check SSL with redirect uri: - url: 'https://httpbin.org/redirect/2' + url: 'https://{{ httpbin_host }}/redirect/2' register: result - name: Assert SSL with redirect assert: that: - - 'result.url|default("") == "https://httpbin.org/get"' + - 'result.url|default("") == "https://{{ httpbin_host }}/get"' - name: redirect to bad SSL site uri: - url: 'http://wrong.host.badssl.com' + url: 'http://{{ badssl_host }}' register: result ignore_errors: true @@ -173,30 +164,30 @@ assert: that: - result|failed - - '"wrong.host.badssl.com" in result.msg' + - 'badssl_host in result.msg' - name: test basic auth uri: - url: 'http://httpbin.org/basic-auth/user/passwd' + url: 'http://{{ httpbin_host }}/basic-auth/user/passwd' user: user password: passwd - name: test basic forced auth uri: - url: 'http://httpbin.org/hidden-basic-auth/user/passwd' + url: 'http://{{ httpbin_host }}/hidden-basic-auth/user/passwd' force_basic_auth: true user: user password: passwd - name: test PUT uri: - url: 'http://httpbin.org/put' + url: 'http://{{ httpbin_host }}/put' method: PUT body: 'foo=bar' - name: test OPTIONS uri: - url: 'http://httpbin.org/' + url: 'http://{{ httpbin_host }}/' method: OPTIONS register: result @@ -217,7 +208,7 @@ # test run. - name: Test that SNI succeeds on python versions that have SNI uri: - url: "{{ SNI_URI }}" + url: 'https://{{ sni_host }}/' return_content: true when: ansible_python.has_sslcontext register: result @@ -225,13 +216,13 @@ - name: Assert SNI verification succeeds on new python assert: that: - - result|success - - "\"

If You Can Read This, You're SNIing

\" in result.content" + - result|success + - 'sni_host == result.content' when: ansible_python.has_sslcontext - name: Verify SNI verification fails on old python without urllib3 contrib uri: - url: '{{ SNI_URI }}' + url: 'https://{{ sni_host }}' ignore_errors: true when: not ansible_python.has_sslcontext register: result @@ -257,7 +248,7 @@ - name: Verify SNI verificaiton succeeds on old python with urllib3 contrib uri: - url: '{{ SNI_URI }}' + url: 'https://{{ sni_host }}' return_content: true when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool register: result @@ -266,8 +257,7 @@ assert: that: - result|success - #- '"Great! Your client" in result.content' - - "\"

If You Can Read This, You're SNIing

\" in result.content" + - 'sni_host == result.content' when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool - name: Uninstall ndg-httpsclient and urllib3 @@ -287,7 +277,7 @@ - name: validate the status_codes are correct uri: - url: https://httpbin.org/status/202 + url: "https://{{ httpbin_host }}/status/202" status_code: 202 method: POST body: foo diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index f58b108386..e2bdef321e 100755 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -4,6 +4,8 @@ set -e set -u set -x +LINKS="--link=httptester:ansible.http.tests --link=httptester:sni1.ansible.http.tests --link=httptester:sni2.ansible.http.tests --link=httptester:fail.ansible.http.tests" + if [ "${TARGET}" = "sanity" ]; then ./test/code-smell/replace-urlopen.sh . ./test/code-smell/use-compat-six.sh lib @@ -12,9 +14,13 @@ if [ "${TARGET}" = "sanity" ]; then if test x"$TOXENV" != x'py24' ; then tox ; fi if test x"$TOXENV" = x'py24' ; then python2.4 -V && python2.4 -m compileall -fq -x 'module_utils/(a10|rax|openstack|ec2|gce|docker_common|azure_rm_common).py' lib/ansible/module_utils ; fi else + if [ ! -e /tmp/cid_httptester ]; then + docker pull sivel/httptester + docker run -d --name=httptester sivel/httptester > /tmp/cid_httptester + fi export C_NAME="testAbull_$$_$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)" docker pull ansible/ansible:${TARGET} - docker run -d --volume="${PWD}:/root/ansible:Z" --name "${C_NAME}" ${TARGET_OPTIONS:=''} ansible/ansible:${TARGET} > /tmp/cid_${TARGET} + docker run -d --volume="${PWD}:/root/ansible:Z" $LINKS --name "${C_NAME}" ${TARGET_OPTIONS:=''} ansible/ansible:${TARGET} > /tmp/cid_${TARGET} docker exec -ti $(cat /tmp/cid_${TARGET}) /bin/sh -c "export TEST_FLAGS='${TEST_FLAGS:-''}'; cd /root/ansible; . hacking/env-setup; (cd test/integration; LC_ALL=en_US.utf-8 make ${MAKE_TARGET:-})" docker kill $(cat /tmp/cid_${TARGET}) From 99e3880181880c8240b18fb1f118d593159101b2 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 10 May 2016 21:56:19 -0700 Subject: [PATCH 055/133] small python3 fix so that ping will run on python3 --- lib/ansible/module_utils/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 2190b3462f..e597b2b407 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1175,7 +1175,7 @@ class AnsibleModule(object): return aliases_results def _check_arguments(self, check_invalid_arguments): - for (k,v) in self.params.items(): + for (k,v) in list(self.params.items()): if k == '_ansible_check_mode' and v: if not self.supports_check_mode: From 9096901c1f740f87a5ecfba86cd5f1a7519a431d Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 10 May 2016 21:59:22 -0700 Subject: [PATCH 056/133] Update submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 1f5cf669dd..9dfed7c849 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 1f5cf669ddec2ea46cd5f335f56f0d493403c99a +Subproject commit 9dfed7c84924ea61b3724b8df2c366ec4bc364cb diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 816673dd6f..2665acb257 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 816673dd6f5df0f9f27753c1c2cadf22ea70befb +Subproject commit 2665acb2578bdcb6edf224a5790b83cfb3bad694 From 33de7707c9027ace26ec62bff8f58717f4080538 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Wed, 11 May 2016 13:17:32 -0400 Subject: [PATCH 057/133] Do not include params when getting role vars in certain situations In VariableManager, we fetch the params specifically in the next step, so including them in the prior step is unnecessary and could lead to things being overridden in an improper order. In Block, we should not be getting the params for the role as they are included earlier via the VariableManager. Fixes #14411 --- lib/ansible/playbook/block.py | 2 +- lib/ansible/vars/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/playbook/block.py b/lib/ansible/playbook/block.py index 1c9569de8d..8b4a6c8a10 100644 --- a/lib/ansible/playbook/block.py +++ b/lib/ansible/playbook/block.py @@ -66,7 +66,7 @@ class Block(Base, Become, Conditional, Taggable): all_vars = self.vars.copy() if self._role: - all_vars.update(self._role.get_vars(self._dep_chain)) + all_vars.update(self._role.get_vars(self._dep_chain, include_params=False)) if self._parent_block: all_vars.update(self._parent_block.get_vars()) if self._task_include: diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py index 4496d20ab5..66b816df96 100644 --- a/lib/ansible/vars/__init__.py +++ b/lib/ansible/vars/__init__.py @@ -324,7 +324,7 @@ class VariableManager: if task: if task._role: - all_vars = combine_vars(all_vars, task._role.get_vars()) + all_vars = combine_vars(all_vars, task._role.get_vars(include_params=False)) all_vars = combine_vars(all_vars, task._role.get_role_params(task._block._dep_chain)) all_vars = combine_vars(all_vars, task.get_vars()) From 18587842c63b16085e405fff8b7e1136310c85d9 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 11 May 2016 12:03:23 -0700 Subject: [PATCH 058/133] Exclude the sample directory from tests --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index f36f5841c5..60b446d955 100644 --- a/tox.ini +++ b/tox.ini @@ -12,10 +12,10 @@ deps = -r{toxinidir}/test/utils/tox/requirements.txt whitelist_externals = make commands = python --version - py26: python -m compileall -fq -x 'test|samples|contrib/inventory/vagrant.py' lib test contrib - py27: python -m compileall -fq -x 'test|samples' lib test contrib - py34: python -m compileall -fq -x 'lib/ansible/module_utils|lib/ansible/modules' lib test contrib - py35: python -m compileall -fq -x 'lib/ansible/module_utils|lib/ansible/modules' lib test contrib + py26: python -m compileall -fq -x 'test/samples|contrib/inventory/vagrant.py' lib test contrib + py27: python -m compileall -fq -x 'test/samples' lib test contrib + py34: python -m compileall -fq -x 'test/samples|lib/ansible/module_utils|lib/ansible/modules' lib test contrib + py35: python -m compileall -fq -x 'test/samples|lib/ansible/module_utils|lib/ansible/modules' lib test contrib make tests From fefd87c61a1cfa7ecc04394c56d45a2b8317d617 Mon Sep 17 00:00:00 2001 From: Carl Date: Tue, 10 May 2016 17:00:07 -0400 Subject: [PATCH 059/133] Fixes #15745 playbook include: Conditional scoping Fixes #15745 Applies conditional forwarding to all tasks/roles within the included playbook. The existing line only applies forwarded conditionals to the main Task block, and misses pre_, post_, and roles. Typo :: Made a selection mistake when I copied over the one line change --- lib/ansible/playbook/playbook_include.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/playbook/playbook_include.py b/lib/ansible/playbook/playbook_include.py index f481ba142c..9cac3317c2 100644 --- a/lib/ansible/playbook/playbook_include.py +++ b/lib/ansible/playbook/playbook_include.py @@ -96,7 +96,7 @@ class PlaybookInclude(Base, Conditional, Taggable): # plays. If so, we can take a shortcut here and simply prepend them to # those attached to each block (if any) if forward_conditional: - for task_block in entry.tasks: + for task_block in entry.pre_tasks + entry.roles + entry.tasks + entry.post_tasks: task_block.when = self.when[:] + task_block.when return pb From 9d9a451b3459fad615a87982dce7d2e68cecf29e Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Wed, 11 May 2016 19:10:30 -0400 Subject: [PATCH 060/133] Simply being in an always block shouldn't mean a host did not fail Previously the changed code was necessary, however it is now problematic as we've started using the is_failed() method in other places in the code. Additional changes at the strategy layer should make this safe to remove now. Fixes #15625 --- lib/ansible/executor/play_iterator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/ansible/executor/play_iterator.py b/lib/ansible/executor/play_iterator.py index 9ecda9395c..7fa396307b 100644 --- a/lib/ansible/executor/play_iterator.py +++ b/lib/ansible/executor/play_iterator.py @@ -475,8 +475,7 @@ class PlayIterator: if state is None: return False elif state.fail_state != self.FAILED_NONE: - if state.run_state == self.ITERATING_RESCUE and state.fail_state&self.FAILED_RESCUE == 0 or \ - state.run_state == self.ITERATING_ALWAYS and state.fail_state&self.FAILED_ALWAYS == 0: + if state.run_state == self.ITERATING_RESCUE and state.fail_state&self.FAILED_RESCUE == 0: return False else: return True From 8a184381afd2756ea10778decb495191e97e5ccc Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 11 May 2016 17:54:01 -0700 Subject: [PATCH 061/133] Strip junk after JSON return. (#15822) Fixes #15601 --- lib/ansible/plugins/action/__init__.py | 52 ++++++++++++++++++------ test/units/plugins/action/test_action.py | 41 +++++++++++++++++++ 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 9a0e99fb8d..7a8ee2b4bc 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -324,7 +324,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): # contain a path to a tmp dir but doesn't know if it needs to # exist or not. If there's no path, then there's no need for us # to do work - self._display.debug('_fixup_perms called with remote_path==None. Sure this is correct?') + display.debug('_fixup_perms called with remote_path==None. Sure this is correct?') return remote_path if self._play_context.become and self._play_context.become_user not in ('root', remote_user): @@ -360,7 +360,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): if C.ALLOW_WORLD_READABLE_TMPFILES: # fs acls failed -- do things this insecure way only # if the user opted in in the config file - self._display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user') + display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user') res = self._remote_chmod('a+%s' % mode, remote_path, recursive=recursive) if res['rc'] != 0: raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr'])) @@ -480,21 +480,49 @@ class ActionBase(with_metaclass(ABCMeta, object)): else: return initial_fragment - def _filter_leading_non_json_lines(self, data): + @staticmethod + def _filter_non_json_lines(data): ''' Used to avoid random output from SSH at the top of JSON output, like messages from tcagetattr, or where dropbear spews MOTD on every single command (which is nuts). - need to filter anything which starts not with '{', '[', ', '=' or is an empty line. - filter only leading lines since multiline JSON is valid. + need to filter anything which does not start with '{', '[', or is an empty line. + Have to be careful how we filter trailing junk as multiline JSON is valid. ''' - idx = 0 - for line in data.splitlines(True): - if line.startswith((u'{', u'[')): + # Filter initial junk + lines = data.splitlines() + for start, line in enumerate(lines): + line = line.strip() + if line.startswith(u'{'): + endchar = u'}' break - idx = idx + len(line) + elif line.startswith(u'['): + endchar = u']' + break + else: + display.debug('No start of json char found') + raise ValueError('No start of json char found') - return data[idx:] + # Filter trailing junk + lines = lines[start:] + lines.reverse() + for end, line in enumerate(lines): + if line.strip().endswith(endchar): + break + else: + display.debug('No end of json char found') + raise ValueError('No end of json char found') + + if end < len(lines) - 1: + # Trailing junk is uncommon and can point to things the user might + # want to change. So print a warning if we find any + trailing_junk = lines[:end] + trailing_junk.reverse() + display.warning('Module invocation had junk after the JSON data: %s' % '\n'.join(trailing_junk)) + + lines = lines[end:] + lines.reverse() + return '\n'.join(lines) def _strip_success_message(self, data): ''' @@ -539,7 +567,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): module_args['_ansible_diff'] = self._play_context.diff # let module know our verbosity - module_args['_ansible_verbosity'] = self._display.verbosity + module_args['_ansible_verbosity'] = display.verbosity (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) if not shebang: @@ -627,7 +655,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): def _parse_returned_data(self, res): try: - data = json.loads(self._filter_leading_non_json_lines(res.get('stdout', u''))) + data = json.loads(self._filter_non_json_lines(res.get('stdout', u''))) except ValueError: # not valid json, lets try to capture error data = dict(failed=True, parsed=False) diff --git a/test/units/plugins/action/test_action.py b/test/units/plugins/action/test_action.py index e9b608790f..2a9bb55693 100644 --- a/test/units/plugins/action/test_action.py +++ b/test/units/plugins/action/test_action.py @@ -33,6 +33,8 @@ try: except ImportError: import __builtin__ as builtins +from nose.tools import eq_, raises + from ansible.release import __version__ as ansible_version from ansible import constants as C from ansible.compat.six import text_type @@ -630,3 +632,42 @@ class TestActionBase(unittest.TestCase): play_context.make_become_cmd.assert_called_once_with("ECHO SAME", executable=None) finally: C.BECOME_ALLOW_SAME_USER = become_allow_same_user + +# Note: Using nose's generator test cases here so we can't inherit from +# unittest.TestCase +class TestFilterNonJsonLines(object): + parsable_cases = ( + (u'{"hello": "world"}', u'{"hello": "world"}'), + (u'{"hello": "world"}\n', u'{"hello": "world"}'), + (u'{"hello": "world"} ', u'{"hello": "world"} '), + (u'{"hello": "world"} \n', u'{"hello": "world"} '), + (u'Message of the Day\n{"hello": "world"}', u'{"hello": "world"}'), + (u'{"hello": "world"}\nEpilogue', u'{"hello": "world"}'), + (u'Several\nStrings\nbefore\n{"hello": "world"}\nAnd\nAfter\n', u'{"hello": "world"}'), + (u'{"hello": "world",\n"olĂ¡": "mundo"}', u'{"hello": "world",\n"olĂ¡": "mundo"}'), + (u'\nPrecedent\n{"hello": "world",\n"olĂ¡": "mundo"}\nAntecedent', u'{"hello": "world",\n"olĂ¡": "mundo"}'), + ) + + unparsable_cases = ( + u'No json here', + u'"olĂ¡": "mundo"', + u'{"No json": "ending"', + u'{"wrong": "ending"]', + u'["wrong": "ending"}', + ) + + def check_filter_non_json_lines(self, stdout_line, parsed): + eq_(parsed, ActionBase._filter_non_json_lines(stdout_line)) + + def test_filter_non_json_lines(self): + for stdout_line, parsed in self.parsable_cases: + yield self.check_filter_non_json_lines, stdout_line, parsed + + @raises(ValueError) + def check_unparsable_filter_non_json_lines(self, stdout_line): + ActionBase._filter_non_json_lines(stdout_line) + + def test_unparsable_filter_non_json_lines(self): + for stdout_line in self.unparsable_cases: + yield self.check_unparsable_filter_non_json_lines, stdout_line + From 0d7530e6daad519df6c6b421ad457b2d6c8573c8 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Wed, 11 May 2016 21:49:46 -0400 Subject: [PATCH 062/133] Change error about loop variable in use to a warning --- lib/ansible/executor/task_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 2a475c2a10..ec220dd7c9 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -232,7 +232,7 @@ class TaskExecutor: loop_var = self._task.loop_control.loop_var or 'item' if loop_var in task_vars: - raise AnsibleError("the loop variable '%s' is already in use. You should set the `loop_var` value in the `loop_control` option for the task to something else to avoid variable collisions" % loop_var) + display.warning("The loop variable '%s' is already in use. You should set the `loop_var` value in the `loop_control` option for the task to something else to avoid variable collisions and unexpected behavior." % loop_var) items = self._squash_items(items, loop_var, task_vars) for item in items: From d391c53b4fda2995cd04890fbda8a05800f21ce4 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Thu, 12 May 2016 10:38:53 -0400 Subject: [PATCH 063/133] Clear blocked hosts when a role duplicate task is found in free strategy In the free strategy, we mark a host as blocked when it has work to do (the PlayIterator returns a task) to prevent multiple tasks from being sent to the host. However, we check for role duplicates after setting the blocked flag, but were not clearing that when the task was skipped leading to an infinite loop. This patch corrects that by clearing the blocked flag when the task is skipped. Fixes #15681 --- lib/ansible/plugins/strategy/free.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ansible/plugins/strategy/free.py b/lib/ansible/plugins/strategy/free.py index fb0ae7dfcc..ae8370d5fb 100644 --- a/lib/ansible/plugins/strategy/free.py +++ b/lib/ansible/plugins/strategy/free.py @@ -123,6 +123,7 @@ class StrategyModule(StrategyBase): # if there is metadata, check to see if the allow_duplicates flag was set to true if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: display.debug("'%s' skipped because role has already run" % task) + del self._blocked_hosts[host_name] continue if task.action == 'meta': From dac356466ca6a59d0ae5452a8da059a09b8a9639 Mon Sep 17 00:00:00 2001 From: Vic Iglesias Date: Thu, 12 May 2016 08:57:26 -0700 Subject: [PATCH 064/133] Update GCE module to use JSON credentials (#13623) * Update GCE module to use JSON credentials * Ensure minimum libcloud version when using JSON crednetials for GCE * Relax langauge around libcloud requirements --- docsite/rst/guide_gce.rst | 70 ++++++++++++++++++++++----------- lib/ansible/module_utils/gce.py | 31 +++++++++++++-- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/docsite/rst/guide_gce.rst b/docsite/rst/guide_gce.rst index c689632818..3a9acabc97 100644 --- a/docsite/rst/guide_gce.rst +++ b/docsite/rst/guide_gce.rst @@ -11,7 +11,7 @@ Introduction Ansible contains modules for managing Google Compute Engine resources, including creating instances, controlling network access, working with persistent disks, and managing load balancers. Additionally, there is an inventory plugin that can automatically suck down all of your GCE instances into Ansible dynamic inventory, and create groups by tag and other properties. -The GCE modules all require the apache-libcloud module, which you can install from pip: +The GCE modules all require the apache-libcloud module which you can install from pip: .. code-block:: bash @@ -22,16 +22,19 @@ The GCE modules all require the apache-libcloud module, which you can install fr Credentials ----------- -To work with the GCE modules, you'll first need to get some credentials. You can create new one from the `console `_ by going to the "APIs and Auth" section and choosing to create a new client ID for a service account. Once you've created a new client ID and downloaded (you must click **Generate new P12 Key**) the generated private key (in the `pkcs12 format `_), you'll need to convert the key by running the following command: +To work with the GCE modules, you'll first need to get some credentials in the +JSON format: -.. code-block:: bash +1. `Create a Service Account `_ +2. `Download JSON credentials `_ - $ openssl pkcs12 -in pkey.pkcs12 -passin pass:notasecret -nodes -nocerts | openssl rsa -out pkey.pem +There are three different ways to provide credentials to Ansible so that it can talk with Google Cloud for provisioning and configuration actions: -There are two different ways to provide credentials to Ansible so that it can talk with Google Cloud for provisioning and configuration actions: +.. note:: If you would like to use JSON credentials you must have libcloud >= 0.17.0 * by providing to the modules directly * by populating a ``secrets.py`` file +* by setting environment variables Calling Modules By Passing Credentials `````````````````````````````````````` @@ -39,7 +42,7 @@ Calling Modules By Passing Credentials For the GCE modules you can specify the credentials as arguments: * ``service_account_email``: email associated with the project -* ``pem_file``: path to the pem file +* ``credentials_file``: path to the JSON credentials file * ``project_id``: id of the project For example, to create a new instance using the cloud module, you can use the following configuration: @@ -48,12 +51,12 @@ For example, to create a new instance using the cloud module, you can use the fo - name: Create instance(s) hosts: localhost - connection: local + connection: local gather_facts: no vars: service_account_email: unique-id@developer.gserviceaccount.com - pem_file: /path/to/project.pem + credentials_file: /path/to/project.json project_id: project-id machine_type: n1-standard-1 image: debian-7 @@ -61,28 +64,50 @@ For example, to create a new instance using the cloud module, you can use the fo tasks: - name: Launch instances - gce: - instance_names: dev + gce: + instance_names: dev machine_type: "{{ machine_type }}" image: "{{ image }}" service_account_email: "{{ service_account_email }}" - pem_file: "{{ pem_file }}" + credentials_file: "{{ credentials_file }}" project_id: "{{ project_id }}" -Calling Modules with secrets.py -``````````````````````````````` +When running Ansible inside a GCE VM you can use the service account credentials from the local metadata server by +setting both ``service_account_email`` and ``credentials_file`` to a blank string. + +Configuring Modules with secrets.py +``````````````````````````````````` Create a file ``secrets.py`` looking like following, and put it in some folder which is in your ``$PYTHONPATH``: .. code-block:: python - GCE_PARAMS = ('i...@project.googleusercontent.com', '/path/to/project.pem') + GCE_PARAMS = ('i...@project.googleusercontent.com', '/path/to/project.json') GCE_KEYWORD_PARAMS = {'project': 'project_id'} Ensure to enter the email address from the created services account and not the one from your main account. Now the modules can be used as above, but the account information can be omitted. +If you are running Ansible from inside a GCE VM with an authorized service account you can set the email address and +credentials path as follows so that get automatically picked up: + +.. code-block:: python + + GCE_PARAMS = ('', '') + GCE_KEYWORD_PARAMS = {'project': 'project_id'} + +Configuring Modules with Environment Variables +`````````````````````````````````````````````` + +Set the following environment variables before running Ansible in order to configure your credentials: + +.. code-block:: bash + + GCE_EMAIL + GCE_PROJECT + GCE_CREDENTIALS_FILE_PATH + GCE Dynamic Inventory --------------------- @@ -171,7 +196,7 @@ A playbook would looks like this: machine_type: n1-standard-1 # default image: debian-7 service_account_email: unique-id@developer.gserviceaccount.com - pem_file: /path/to/project.pem + credentials_file: /path/to/project.json project_id: project-id tasks: @@ -181,7 +206,7 @@ A playbook would looks like this: machine_type: "{{ machine_type }}" image: "{{ image }}" service_account_email: "{{ service_account_email }}" - pem_file: "{{ pem_file }}" + credentials_file: "{{ credentials_file }}" project_id: "{{ project_id }}" tags: webserver register: gce @@ -224,7 +249,7 @@ a basic example of what is possible:: machine_type: n1-standard-1 # default image: debian-7 service_account_email: unique-id@developer.gserviceaccount.com - pem_file: /path/to/project.pem + credentials_file: /path/to/project.json project_id: project-id roles: @@ -238,13 +263,12 @@ a basic example of what is possible:: args: fwname: "all-http" name: "default" - allowed: "tcp:80" - state: "present" - service_account_email: "{{ service_account_email }}" - pem_file: "{{ pem_file }}" + allowed: "tcp:80" + state: "present" + service_account_email: "{{ service_account_email }}" + credentials_file: "{{ credentials_file }}" project_id: "{{ project_id }}" By pointing your browser to the IP of the server, you should see a page welcoming you. -Upgrades to this documentation are welcome, hit the github link at the top right of this page if you would like to make additions! - +Upgrades to this documentation are welcome, hit the github link at the top right of this page if you would like to make additions! diff --git a/lib/ansible/module_utils/gce.py b/lib/ansible/module_utils/gce.py index 5f222739aa..6bfa40798a 100644 --- a/lib/ansible/module_utils/gce.py +++ b/lib/ansible/module_utils/gce.py @@ -27,10 +27,14 @@ # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # +import json import os import traceback from libcloud.compute.types import Provider +from distutils.version import LooseVersion + +import libcloud from libcloud.compute.providers import get_driver USER_AGENT_PRODUCT="Ansible-gce" @@ -39,6 +43,7 @@ USER_AGENT_VERSION="v1" def gce_connect(module, provider=None): """Return a Google Cloud Engine connection.""" service_account_email = module.params.get('service_account_email', None) + credentials_file = module.params.get('credentials_file', None) pem_file = module.params.get('pem_file', None) project_id = module.params.get('project_id', None) @@ -50,6 +55,8 @@ def gce_connect(module, provider=None): project_id = os.environ.get('GCE_PROJECT', None) if not pem_file: pem_file = os.environ.get('GCE_PEM_FILE_PATH', None) + if not credentials_file: + credentials_file = os.environ.get('GCE_CREDENTIALS_FILE_PATH', pem_file) # If we still don't have one or more of our credentials, attempt to # get the remaining values from the libcloud secrets file. @@ -62,25 +69,41 @@ def gce_connect(module, provider=None): if hasattr(secrets, 'GCE_PARAMS'): if not service_account_email: service_account_email = secrets.GCE_PARAMS[0] - if not pem_file: - pem_file = secrets.GCE_PARAMS[1] + if not credentials_file: + credentials_file = secrets.GCE_PARAMS[1] keyword_params = getattr(secrets, 'GCE_KEYWORD_PARAMS', {}) if not project_id: project_id = keyword_params.get('project', None) # If we *still* don't have the credentials we need, then it's time to # just fail out. - if service_account_email is None or pem_file is None or project_id is None: + if service_account_email is None or credentials_file is None or project_id is None: module.fail_json(msg='Missing GCE connection parameters in libcloud ' 'secrets file.') return None + else: + # We have credentials but lets make sure that if they are JSON we have the minimum + # libcloud requirement met + try: + # Try to read credentials as JSON + with open(credentials_file) as credentials: + json.loads(credentials.read()) + # If the credentials are proper JSON and we do not have the minimum + # required libcloud version, bail out and return a descriptive error + if LooseVersion(libcloud.__version__) < '0.17.0': + module.fail_json(msg='Using JSON credentials but libcloud minimum version not met. ' + 'Upgrade to libcloud>=0.17.0.') + return None + except ValueError, e: + # Not JSON + pass # Allow for passing in libcloud Google DNS (e.g, Provider.GOOGLE) if provider is None: provider = Provider.GCE try: - gce = get_driver(provider)(service_account_email, pem_file, + gce = get_driver(provider)(service_account_email, credentials_file, datacenter=module.params.get('zone', None), project=project_id) gce.connection.user_agent_append("%s/%s" % ( From 4d59779e0adf35b8710f20d966c527ea7efab466 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 12 May 2016 09:13:46 -0700 Subject: [PATCH 065/133] Fixed importing the libcloud modules to give a nice error rather than a traceback. --- lib/ansible/module_utils/gce.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/ansible/module_utils/gce.py b/lib/ansible/module_utils/gce.py index 6bfa40798a..a57fdb46d5 100644 --- a/lib/ansible/module_utils/gce.py +++ b/lib/ansible/module_utils/gce.py @@ -30,18 +30,24 @@ import json import os import traceback - -from libcloud.compute.types import Provider from distutils.version import LooseVersion -import libcloud -from libcloud.compute.providers import get_driver +try: + from libcloud.compute.types import Provider + import libcloud + from libcloud.compute.providers import get_driver + HAS_LIBCLOUD_BASE = True +except ImportError: + HAS_LIBCLOUD_BASE = False USER_AGENT_PRODUCT="Ansible-gce" USER_AGENT_VERSION="v1" def gce_connect(module, provider=None): """Return a Google Cloud Engine connection.""" + if not HAS_LIBCLOUD_BASE: + module.fail_json(msg='libcloud must be installed to use this module') + service_account_email = module.params.get('service_account_email', None) credentials_file = module.params.get('credentials_file', None) pem_file = module.params.get('pem_file', None) From 1cee3f35b158ecb3a1d61532aa4fb8a57c1a8bd7 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 8 Apr 2016 10:18:35 -0500 Subject: [PATCH 066/133] Guard against a shell profile printing extraneous data --- lib/ansible/module_utils/basic.py | 14 +++++++------- lib/ansible/plugins/action/__init__.py | 3 ++- lib/ansible/plugins/shell/__init__.py | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index e597b2b407..6e20d54bee 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -136,10 +136,10 @@ except ImportError: try: import simplejson as json except ImportError: - print('{"msg": "Error: ansible requires the stdlib json or simplejson module, neither was found!", "failed": true}') + print('\n{"msg": "Error: ansible requires the stdlib json or simplejson module, neither was found!", "failed": true}') sys.exit(1) except SyntaxError: - print('{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}') + print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}') sys.exit(1) HAVE_SELINUX=False @@ -574,7 +574,7 @@ class AnsibleModule(object): except Exception: e = get_exception() # Use exceptions here because it isn't safe to call fail_json until no_log is processed - print('{"failed": true, "msg": "Module alias error: %s"}' % str(e)) + print('\n{"failed": true, "msg": "Module alias error: %s"}' % str(e)) sys.exit(1) # Save parameter values that should never be logged @@ -1497,7 +1497,7 @@ class AnsibleModule(object): params = json.loads(buffer.decode('utf-8')) except ValueError: # This helper used too early for fail_json to work. - print('{"msg": "Error: Module unable to decode valid JSON on stdin. Unable to figure out what parameters were passed", "failed": true}') + print('\n{"msg": "Error: Module unable to decode valid JSON on stdin. Unable to figure out what parameters were passed", "failed": true}') sys.exit(1) if sys.version_info < (3,): @@ -1508,7 +1508,7 @@ class AnsibleModule(object): self.constants = params['ANSIBLE_MODULE_CONSTANTS'] except KeyError: # This helper used too early for fail_json to work. - print('{"msg": "Error: Module unable to locate ANSIBLE_MODULE_ARGS and ANSIBLE_MODULE_CONSTANTS in json data from stdin. Unable to figure out what parameters were passed", "failed": true}') + print('\n{"msg": "Error: Module unable to locate ANSIBLE_MODULE_ARGS and ANSIBLE_MODULE_CONSTANTS in json data from stdin. Unable to figure out what parameters were passed", "failed": true}') sys.exit(1) def _log_to_syslog(self, msg): @@ -1700,7 +1700,7 @@ class AnsibleModule(object): kwargs['invocation'] = {'module_args': self.params} kwargs = remove_values(kwargs, self.no_log_values) self.do_cleanup_files() - print(self.jsonify(kwargs)) + print('\n%s' % self.jsonify(kwargs)) sys.exit(0) def fail_json(self, **kwargs): @@ -1712,7 +1712,7 @@ class AnsibleModule(object): kwargs['invocation'] = {'module_args': self.params} kwargs = remove_values(kwargs, self.no_log_values) self.do_cleanup_files() - print(self.jsonify(kwargs)) + print('\n%s' % self.jsonify(kwargs)) sys.exit(1) def fail_on_missing_params(self, required_params=None): diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 7a8ee2b4bc..5ac2aba59d 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -240,7 +240,8 @@ class ActionBase(with_metaclass(ABCMeta, object)): raise AnsibleConnectionFailure(output) try: - rc = self._connection._shell.join_path(result['stdout'].strip(), u'').splitlines()[-1] + stdout_parts = result['stdout'].strip().split('%s=' % basefile, 1) + rc = self._connection._shell.join_path(stdout_parts[-1], u'').splitlines()[-1] except IndexError: # stdout was empty or just space, set to / to trigger error in next if rc = '/' diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py index effbddd58e..6b0ed98d1d 100644 --- a/lib/ansible/plugins/shell/__init__.py +++ b/lib/ansible/plugins/shell/__init__.py @@ -134,7 +134,7 @@ class ShellBase(object): basetmp = self.join_path(basetmpdir, basefile) cmd = 'mkdir -p %s echo %s %s' % (self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) - cmd += ' %s echo %s echo %s %s' % (self._SHELL_AND, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) + cmd += ' %s echo %s=%s echo %s %s' % (self._SHELL_AND, basefile, self._SHELL_SUB_LEFT, basetmp, self._SHELL_SUB_RIGHT) # change the umask in a subshell to achieve the desired mode # also for directories created with `mkdir -p` From 292f0ed0d627a486ec376d649f3ae73f21d27354 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 11 May 2016 18:26:39 -0700 Subject: [PATCH 067/133] If we can't squash for any reason, then simply do not optimize the items loop. Also add more squashing testcases Fixes #15649 --- lib/ansible/executor/task_executor.py | 103 ++++++++++++---------- test/units/executor/test_task_executor.py | 102 ++++++++++++++------- 2 files changed, 124 insertions(+), 81 deletions(-) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 2a475c2a10..d97fdf41f6 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -269,59 +269,64 @@ class TaskExecutor: Squash items down to a comma-separated list for certain modules which support it (typically package management modules). ''' - # _task.action could contain templatable strings (via action: and - # local_action:) Template it before comparing. If we don't end up - # optimizing it here, the templatable string might use template vars - # that aren't available until later (it could even use vars from the - # with_items loop) so don't make the templated string permanent yet. - templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=variables) - task_action = self._task.action - if templar._contains_vars(task_action): - task_action = templar.template(task_action, fail_on_undefined=False) + try: + # _task.action could contain templatable strings (via action: and + # local_action:) Template it before comparing. If we don't end up + # optimizing it here, the templatable string might use template vars + # that aren't available until later (it could even use vars from the + # with_items loop) so don't make the templated string permanent yet. + templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=variables) + task_action = self._task.action + if templar._contains_vars(task_action): + task_action = templar.template(task_action, fail_on_undefined=False) - if len(items) > 0 and task_action in self.SQUASH_ACTIONS: - if all(isinstance(o, string_types) for o in items): - final_items = [] + if len(items) > 0 and task_action in self.SQUASH_ACTIONS: + if all(isinstance(o, string_types) for o in items): + final_items = [] - name = None - for allowed in ['name', 'pkg', 'package']: - name = self._task.args.pop(allowed, None) - if name is not None: - break + name = None + for allowed in ['name', 'pkg', 'package']: + name = self._task.args.pop(allowed, None) + if name is not None: + break - # This gets the information to check whether the name field - # contains a template that we can squash for - template_no_item = template_with_item = None - if name: - if templar._contains_vars(name): - variables[loop_var] = '\0$' - template_no_item = templar.template(name, variables, cache=False) - variables[loop_var] = '\0@' - template_with_item = templar.template(name, variables, cache=False) - del variables[loop_var] + # This gets the information to check whether the name field + # contains a template that we can squash for + template_no_item = template_with_item = None + if name: + if templar._contains_vars(name): + variables[loop_var] = '\0$' + template_no_item = templar.template(name, variables, cache=False) + variables[loop_var] = '\0@' + template_with_item = templar.template(name, variables, cache=False) + del variables[loop_var] - # Check if the user is doing some operation that doesn't take - # name/pkg or the name/pkg field doesn't have any variables - # and thus the items can't be squashed - if template_no_item != template_with_item: - for item in items: - variables[loop_var] = item - if self._task.evaluate_conditional(templar, variables): - new_item = templar.template(name, cache=False) - final_items.append(new_item) - self._task.args['name'] = final_items - # Wrap this in a list so that the calling function loop - # executes exactly once - return [final_items] - else: - # Restore the name parameter - self._task.args['name'] = name - #elif: - # Right now we only optimize single entries. In the future we - # could optimize more types: - # * lists can be squashed together - # * dicts could squash entries that match in all cases except the - # name or pkg field. + # Check if the user is doing some operation that doesn't take + # name/pkg or the name/pkg field doesn't have any variables + # and thus the items can't be squashed + if template_no_item != template_with_item: + for item in items: + variables[loop_var] = item + if self._task.evaluate_conditional(templar, variables): + new_item = templar.template(name, cache=False) + final_items.append(new_item) + self._task.args['name'] = final_items + # Wrap this in a list so that the calling function loop + # executes exactly once + return [final_items] + else: + # Restore the name parameter + self._task.args['name'] = name + #elif: + # Right now we only optimize single entries. In the future we + # could optimize more types: + # * lists can be squashed together + # * dicts could squash entries that match in all cases except the + # name or pkg field. + except: + # Squashing is an optimization. If it fails for any reason, + # simply use the unoptimized list of items. + pass return items def _execute(self, variables=None): diff --git a/test/units/executor/test_task_executor.py b/test/units/executor/test_task_executor.py index 8f2c1ed8cc..87704e2188 100644 --- a/test/units/executor/test_task_executor.py +++ b/test/units/executor/test_task_executor.py @@ -180,8 +180,10 @@ class TestTaskExecutor(unittest.TestCase): mock_host = MagicMock() + loop_var = 'item' + def _evaluate_conditional(templar, variables): - item = variables.get('item') + item = variables.get(loop_var) if item == 'b': return False return True @@ -230,9 +232,31 @@ class TestTaskExecutor(unittest.TestCase): new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) + mock_task.action = '{{unknown}}' + mock_task.args={'name': '{{item}}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + self.assertEqual(new_items, ['a', 'b', 'c']) + + # Maybe should raise an error in this case. The user would have to specify: + # - yum: name="{{ packages[item] }}" + # with_items: + # - ['a', 'b'] + # - ['foo', 'bar'] + # you can't use a list as a dict key so that would probably throw + # an error later. If so, we can throw it now instead. + # Squashing in this case would not be intuitive as the user is being + # explicit in using each list entry as a key. + job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "foo": "baz", "bar": "quux" }) + items = [['a', 'b'], ['foo', 'bar']] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ packages[item] }}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + self.assertEqual(new_items, items) + # # Replaces # + items = ['a', 'b', 'c'] mock_task.action = 'yum' mock_task.args={'name': '{{item}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) @@ -243,29 +267,65 @@ class TestTaskExecutor(unittest.TestCase): new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, [['a', 'c']]) + # New loop_var + mock_task.action = 'yum' + mock_task.args = {'name': '{{a_loop_var_item}}'} + mock_task.loop_control = {'loop_var': 'a_loop_var_item'} + loop_var = 'a_loop_var_item' + new_items = te._squash_items(items=items, loop_var='a_loop_var_item', variables=job_vars) + self.assertEqual(new_items, [['a', 'c']]) + loop_var = 'item' + # - # Smoketests -- these won't optimize but make sure that they don't - # traceback either + # These are presently not optimized but could be in the future. + # Expected output if they were optimized is given as a comment + # Please move these to a different section if they are optimized # - mock_task.action = '{{unknown}}' - mock_task.args={'name': '{{item}}'} + + # Squashing lists + job_vars = dict(pkg_mgr='yum') + items = [['a', 'b'], ['foo', 'bar']] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ item }}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) - self.assertEqual(new_items, ['a', 'b', 'c']) + #self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']]) + self.assertEqual(new_items, items) + + # Retrieving from a dict + items = ['a', 'b', 'foo'] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ packages[item] }}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + #self.assertEqual(new_items, [['foo', 'baz']]) + self.assertEqual(new_items, items) + + # Another way to retrieve from a dict + job_vars = dict(pkg_mgr='yum') + items = [{'package': 'foo'}, {'package': 'bar'}] + mock_task.action = 'yum' + mock_task.args = {'name': '{{ item["package"] }}'} + new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) + #self.assertEqual(new_items, [['foo', 'bar']]) + self.assertEqual(new_items, items) items = [dict(name='a', state='present'), dict(name='b', state='present'), dict(name='c', state='present')] mock_task.action = 'yum' - mock_task.args={'name': '{{item}}'} + mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, items) + #self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')]) - job_vars = dict(pkg_mgr='yum') - items = [{'package': 'foo'}, {'package': 'bar'}] + items = [dict(name='a', state='present'), + dict(name='b', state='present'), + dict(name='c', state='absent')] mock_task.action = 'yum' - mock_task.args = {'name': '{{ items["package"] }}'} + mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, items) + #self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'), + # dict(name='c', state='absent')]) # Could do something like this to recover from bad deps in a package job_vars = dict(pkg_mgr='yum', packages=['a', 'b']) @@ -275,28 +335,6 @@ class TestTaskExecutor(unittest.TestCase): new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, items) - # - # These are presently not optimized but could be in the future. - # Expected output if they were optimized is given as a comment - # Please move these to a different section if they are optimized - # - - job_vars = dict(pkg_mgr='yum') - items = [['a', 'b'], ['foo', 'bar']] - mock_task.action = 'yum' - mock_task.args = {'name': '{{ packages[item] }}'} - new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) - self.assertEqual(new_items, items) - #self.assertEqual(new_items, ['a', 'b', 'foo', 'bar']) - - ### Enable this when we've fixed https://github.com/ansible/ansible/issues/15649 - #job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "c": "baz" }) - #items = ['a', 'b', 'c'] - #mock_task.action = 'yum' - #mock_task.args = {'name': '{{ packages[item] }}'} - #new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) - #self.assertEqual(new_items, items) - #self.assertEqual(new_items, ['foo', 'bar', 'baz']) def test_task_executor_execute(self): fake_loader = DictDataLoader({}) From d8a243bef0a06c20503db70db4e0c955a4e49734 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 8 Jan 2016 10:34:46 -0600 Subject: [PATCH 068/133] First pass at allowing binary modules --- lib/ansible/plugins/action/__init__.py | 24 +++++++++++++++++++----- lib/ansible/plugins/action/async.py | 9 ++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 7a8ee2b4bc..57e4c2fa40 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -100,6 +100,12 @@ class ActionBase(with_metaclass(ABCMeta, object)): return True return False + def _is_binary(self, module_path): + textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f}) + + with open(module_path, 'rb') as f: + start = f.read(1024) + def _configure_module(self, module_name, module_args, task_vars=None): ''' Handles the loading and templating of the module code through the @@ -147,7 +153,10 @@ class ActionBase(with_metaclass(ABCMeta, object)): # insert shared code and arguments into the module (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) - return (module_style, module_shebang, module_data) + if self._is_binary(module_path): + return ('non_native_want_json', None, module_path, True) + + return (module_style, module_shebang, module_data, False) def _compute_environment_string(self): ''' @@ -569,8 +578,9 @@ class ActionBase(with_metaclass(ABCMeta, object)): # let module know our verbosity module_args['_ansible_verbosity'] = display.verbosity - (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) - if not shebang: + (module_style, shebang, module_data, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + + if not shebang and not is_binary: raise AnsibleError("module (%s) is missing interpreter line" % module_name) # a remote tmp path may be necessary and not already created @@ -588,7 +598,11 @@ class ActionBase(with_metaclass(ABCMeta, object)): if remote_module_path or module_style != 'new': display.debug("transferring module to remote") - self._transfer_data(remote_module_path, module_data) + if is_binary: + # If is_binary module_data is the path to the module to transfer + self._transfer_file(module_data, remote_module_path) + else: + self._transfer_data(remote_module_path, module_data, is_path=is_binary) if module_style == 'old': # we need to dump the module args to a k=v string in a file on # the remote system, which can be read and parsed by the module @@ -604,7 +618,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): # Fix permissions of the tmp path and tmp files. This should be # called after all files have been transferred. - self._fixup_perms(tmp, remote_user, recursive=True) + self._fixup_perms(tmp, remote_user, recursive=True, execute=is_binary) cmd = "" in_data = None diff --git a/lib/ansible/plugins/action/async.py b/lib/ansible/plugins/action/async.py index 4f7aa634ce..f6d1a24c81 100644 --- a/lib/ansible/plugins/action/async.py +++ b/lib/ansible/plugins/action/async.py @@ -54,11 +54,14 @@ class ActionModule(ActionBase): module_args['_ansible_no_log'] = True # configure, upload, and chmod the target module - (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) - self._transfer_data(remote_module_path, module_data) + (module_style, shebang, module_data, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + if is_binary: + self._transfer_file(module_data, remote_module_path) + else: + self._transfer_data(remote_module_path, module_data) # configure, upload, and chmod the async_wrapper module - (async_module_style, shebang, async_module_data) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) + (async_module_style, shebang, async_module_data, is_binary) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) self._transfer_data(async_module_path, async_module_data) argsfile = None From 0a8d016642d022c75927aeb6e1450a995cf31bc5 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 11 Jan 2016 11:07:05 -0600 Subject: [PATCH 069/133] Get binary modules working for windows, assuming .exe for windows --- lib/ansible/plugins/action/__init__.py | 14 +++++++------- lib/ansible/plugins/action/async.py | 6 +++--- lib/ansible/plugins/connection/winrm.py | 2 +- lib/ansible/plugins/shell/powershell.py | 12 +++++++++--- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 57e4c2fa40..5fa5af9884 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -105,6 +105,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): with open(module_path, 'rb') as f: start = f.read(1024) + return bool(start.translate(None, textchars)) def _configure_module(self, module_name, module_args, task_vars=None): ''' @@ -154,9 +155,9 @@ class ActionBase(with_metaclass(ABCMeta, object)): (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) if self._is_binary(module_path): - return ('non_native_want_json', None, module_path, True) + return ('non_native_want_json', None, None, module_path, True) - return (module_style, module_shebang, module_data, False) + return (module_style, module_shebang, module_data, module_path, False) def _compute_environment_string(self): ''' @@ -578,7 +579,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): # let module know our verbosity module_args['_ansible_verbosity'] = display.verbosity - (module_style, shebang, module_data, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + (module_style, shebang, module_data, module_path, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) if not shebang and not is_binary: raise AnsibleError("module (%s) is missing interpreter line" % module_name) @@ -590,7 +591,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): tmp = self._make_tmp_path(remote_user) if tmp: - remote_module_filename = self._connection._shell.get_remote_filename(module_name) + remote_module_filename = self._connection._shell.get_remote_filename(module_path) remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename) if module_style in ['old', 'non_native_want_json']: # we'll also need a temp file to hold our module arguments @@ -599,10 +600,9 @@ class ActionBase(with_metaclass(ABCMeta, object)): if remote_module_path or module_style != 'new': display.debug("transferring module to remote") if is_binary: - # If is_binary module_data is the path to the module to transfer - self._transfer_file(module_data, remote_module_path) + self._transfer_file(module_path, remote_module_path) else: - self._transfer_data(remote_module_path, module_data, is_path=is_binary) + self._transfer_data(remote_module_path, module_data) if module_style == 'old': # we need to dump the module args to a k=v string in a file on # the remote system, which can be read and parsed by the module diff --git a/lib/ansible/plugins/action/async.py b/lib/ansible/plugins/action/async.py index f6d1a24c81..a9406421c2 100644 --- a/lib/ansible/plugins/action/async.py +++ b/lib/ansible/plugins/action/async.py @@ -54,14 +54,14 @@ class ActionModule(ActionBase): module_args['_ansible_no_log'] = True # configure, upload, and chmod the target module - (module_style, shebang, module_data, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + (module_style, shebang, module_data, module_path, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) if is_binary: - self._transfer_file(module_data, remote_module_path) + self._transfer_file(module_path, remote_module_path) else: self._transfer_data(remote_module_path, module_data) # configure, upload, and chmod the async_wrapper module - (async_module_style, shebang, async_module_data, is_binary) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) + (async_module_style, shebang, async_module_data, async_module_path, is_binary) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) self._transfer_data(async_module_path, async_module_data) argsfile = None diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py index f5023d7efc..c5efd481f2 100644 --- a/lib/ansible/plugins/connection/winrm.py +++ b/lib/ansible/plugins/connection/winrm.py @@ -62,7 +62,7 @@ class Connection(ConnectionBase): '''WinRM connections over HTTP/HTTPS.''' transport = 'winrm' - module_implementation_preferences = ('.ps1', '') + module_implementation_preferences = ('.ps1', '.exe', '') become_methods = [] allow_executable = False diff --git a/lib/ansible/plugins/shell/powershell.py b/lib/ansible/plugins/shell/powershell.py index aa77cb5d36..dfbae1b428 100644 --- a/lib/ansible/plugins/shell/powershell.py +++ b/lib/ansible/plugins/shell/powershell.py @@ -55,9 +55,11 @@ class ShellModule(object): return '\'%s\'' % path # powershell requires that script files end with .ps1 - def get_remote_filename(self, base_name): - if not base_name.strip().lower().endswith('.ps1'): - return base_name.strip() + '.ps1' + def get_remote_filename(self, pathname): + base_name = os.path.basename(pathname.strip()) + name, ext = os.path.splitext(base_name.strip()) + if ext.lower() not in ['.ps1', '.exe']: + return name + '.ps1' return base_name.strip() @@ -146,6 +148,10 @@ class ShellModule(object): cmd_parts.insert(0, '&') elif shebang and shebang.startswith('#!'): cmd_parts.insert(0, shebang[2:]) + elif not shebang: + # The module is assumed to be a binary + cmd_parts[0] = self._unquote(cmd_parts[0]) + cmd_parts.append(arg_path) script = ''' Try { From 1e038e5043dd7ce63181eb25f6f7dcbc158e4581 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 11 Jan 2016 11:11:00 -0600 Subject: [PATCH 070/133] Update for py26 --- lib/ansible/plugins/action/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 5fa5af9884..5e7113e867 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -101,7 +101,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): return False def _is_binary(self, module_path): - textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f}) + textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) with open(module_path, 'rb') as f: start = f.read(1024) From 35246abb2e14b2aaa323587671b719151bf2eb84 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 11 Jan 2016 11:33:33 -0600 Subject: [PATCH 071/133] Don't register new vars that aren't needed --- lib/ansible/plugins/action/async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/action/async.py b/lib/ansible/plugins/action/async.py index a9406421c2..b2d9370e9e 100644 --- a/lib/ansible/plugins/action/async.py +++ b/lib/ansible/plugins/action/async.py @@ -61,7 +61,7 @@ class ActionModule(ActionBase): self._transfer_data(remote_module_path, module_data) # configure, upload, and chmod the async_wrapper module - (async_module_style, shebang, async_module_data, async_module_path, is_binary) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) + (async_module_style, shebang, async_module_data, _, _) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) self._transfer_data(async_module_path, async_module_data) argsfile = None From 6ad8ec0919e39903e6674c5f71da9577268ca87d Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 12 Feb 2016 11:56:46 -0600 Subject: [PATCH 072/133] Add integration tests for binary modules --- test/integration/Makefile | 6 ++ test/integration/library/.gitignore | 1 + test/integration/library/helloworld.go | 74 +++++++++++++++++++ .../roles/test_binary_modules/tasks/main.yml | 54 ++++++++++++++ test/integration/test_binary_modules.yml | 6 ++ 5 files changed, 141 insertions(+) create mode 100644 test/integration/library/.gitignore create mode 100644 test/integration/library/helloworld.go create mode 100644 test/integration/roles/test_binary_modules/tasks/main.yml create mode 100644 test/integration/test_binary_modules.yml diff --git a/test/integration/Makefile b/test/integration/Makefile index a812ace890..350dd43e98 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -284,3 +284,9 @@ test_lookup_paths: setup no_log: setup # This test expects 7 loggable vars and 0 non loggable ones, if either mismatches it fails, run the ansible-playbook command to debug [ "$$(ansible-playbook no_log_local.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -vvvvv | awk --source 'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "6/0" ] + +test_binary_modules: + cd library && GOOS=linux GOARCH=amd64 go build -o helloworld_linux helloworld.go + cd library && GOOS=windows GOARCH=amd64 go build -o helloworld_win32nt.exe helloworld.go + cd library && GOOS=darwin GOARCH=amd64 go build -o helloworld_darwin helloworld.go + ansible-playbook test_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS) diff --git a/test/integration/library/.gitignore b/test/integration/library/.gitignore new file mode 100644 index 0000000000..d034a06ac7 --- /dev/null +++ b/test/integration/library/.gitignore @@ -0,0 +1 @@ +helloworld_* diff --git a/test/integration/library/helloworld.go b/test/integration/library/helloworld.go new file mode 100644 index 0000000000..d4d0c82d53 --- /dev/null +++ b/test/integration/library/helloworld.go @@ -0,0 +1,74 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" +) + +type ModuleArgs struct { + Name string +} + +type Response struct { + Msg string `json:"msg"` + Changed bool `json:"changed"` + Failed bool `json:"failed"` +} + +func ExitJson(responseBody Response) { + returnResponse(responseBody) +} + +func FailJson(responseBody Response) { + responseBody.Failed = true + returnResponse(responseBody) +} + +func returnResponse(responseBody Response) { + var response []byte + var err error + response, err = json.Marshal(responseBody) + if err != nil { + response, _ = json.Marshal(Response{Msg: "Invalid response object"}) + } + fmt.Println(string(response)) + if responseBody.Failed { + os.Exit(1) + } else { + os.Exit(0) + } +} + +func main() { + var response Response + + if len(os.Args) != 2 { + response.Msg = "No argument file provided" + FailJson(response) + } + + argsFile := os.Args[1] + + text, err := ioutil.ReadFile(argsFile) + if err != nil { + response.Msg = "Could not read configuration file: " + argsFile + FailJson(response) + } + + var moduleArgs ModuleArgs + err = json.Unmarshal(text, &moduleArgs) + if err != nil { + response.Msg = "Configuration file not valid JSON: " + argsFile + FailJson(response) + } + + var name string = "World" + if moduleArgs.Name != "" { + name = moduleArgs.Name + } + + response.Msg = "Hello, " + name + "!" + ExitJson(response) +} diff --git a/test/integration/roles/test_binary_modules/tasks/main.yml b/test/integration/roles/test_binary_modules/tasks/main.yml new file mode 100644 index 0000000000..e40ea9ded5 --- /dev/null +++ b/test/integration/roles/test_binary_modules/tasks/main.yml @@ -0,0 +1,54 @@ +- debug: var=ansible_system + +- name: ping + ping: + when: ansible_system != 'Win32NT' + +- name: win_ping + win_ping: + when: ansible_system == 'Win32NT' + +- name: Hello, World! + action: "helloworld_{{ ansible_system|lower }}" + register: hello_world + +- assert: + that: + - 'hello_world.msg == "Hello, World!"' + +- name: Hello, Ansible! + action: "helloworld_{{ ansible_system|lower }}" + args: + name: Ansible + register: hello_ansible + +- assert: + that: + - 'hello_ansible.msg == "Hello, Ansible!"' + +- name: Async Hello, World! + action: "helloworld_{{ ansible_system|lower }}" + async: 1 + poll: 1 + when: ansible_system != 'Win32NT' + register: async_hello_world + +- assert: + that: + - 'async_hello_world.msg == "Hello, World!"' + when: not async_hello_world|skipped + +- name: Async Hello, Ansible! + action: "helloworld_{{ ansible_system|lower }}" + args: + name: Ansible + async: 1 + poll: 1 + when: ansible_system != 'Win32NT' + register: async_hello_ansible + +- assert: + that: + - 'async_hello_ansible.msg == "Hello, Ansible!"' + when: not async_hello_ansible|skipped + diff --git a/test/integration/test_binary_modules.yml b/test/integration/test_binary_modules.yml new file mode 100644 index 0000000000..6b1a94e6ed --- /dev/null +++ b/test/integration/test_binary_modules.yml @@ -0,0 +1,6 @@ +- hosts: all + roles: + - role: test_binary_modules + tags: + - test_binary_modules + From 2d18607f1e8bf9d7b91a61df5103f560d26e0e06 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 12 Feb 2016 12:06:01 -0600 Subject: [PATCH 073/133] Add GPL3 header to helloworld.go --- test/integration/library/helloworld.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/integration/library/helloworld.go b/test/integration/library/helloworld.go index d4d0c82d53..a4c16b20e5 100644 --- a/test/integration/library/helloworld.go +++ b/test/integration/library/helloworld.go @@ -1,3 +1,18 @@ +// This file is part of Ansible +// +// Ansible is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Ansible is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Ansible. If not, see . + package main import ( From ddf3c3838feb09b35711e8975e822814fc8b7a00 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 12 Feb 2016 16:10:13 -0600 Subject: [PATCH 074/133] Re-implement/move some code lost due to merge conflicts --- lib/ansible/plugins/shell/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py index effbddd58e..0e951b7591 100644 --- a/lib/ansible/plugins/shell/__init__.py +++ b/lib/ansible/plugins/shell/__init__.py @@ -50,7 +50,8 @@ class ShellBase(object): return os.path.join(*args) # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say - def get_remote_filename(self, base_name): + def get_remote_filename(self, pathname): + base_name = os.path.basename(pathname.strip()) return base_name.strip() def path_has_trailing_slash(self, path): @@ -164,7 +165,12 @@ class ShellBase(object): # don't quote the cmd if it's an empty string, because this will break pipelining mode if cmd.strip() != '': cmd = pipes.quote(cmd) - cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd] + cmd_parts = [] + if shebang: + shebang = shebang.replace("#!", "").strip() + else: + shebang = "" + cmd_parts.extend([env_string.strip(), shebang, cmd]) if arg_path is not None: cmd_parts.append(arg_path) new_cmd = " ".join(cmd_parts) From c22c1b47855b6a3b27f6335a6fd78f3f1e671e48 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 23 Feb 2016 12:14:52 -0600 Subject: [PATCH 075/133] Add note about reading input for binary modules --- docsite/rst/developing_modules.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index d1a2cdf44c..c50536d8e5 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -204,6 +204,24 @@ This should return something like:: {"changed": true, "time": "2012-03-14 12:23:00.000307"} +.. _binary_module_reading_input: + +Binary Modules Input +~~~~~~~~~~~~~~~~~~~~ + +Support for binary modules was added in Ansible 2.1. When Ansible detects a binary module, it will proceed to +supply the argument input as a file on ``argv[1]`` that is formatted as JSON. The JSON contents of that file +would resemble something similar to:: + + { + "name": "Ansible", + "_ansible_verbosity": 4, + "_ansible_diff": false, + "_ansible_debug": false, + "_ansible_check_mode": false, + "_ansible_no_log": false + } + .. _module_provided_facts: Module Provided 'Facts' From a4d2238e502a5d0d8b7d31f80198c21859a44212 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 12 Apr 2016 15:02:29 -0500 Subject: [PATCH 076/133] Bumping binary modules functionality to 2.2 --- docsite/rst/developing_modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index c50536d8e5..f1f5d4a2a2 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -209,7 +209,7 @@ This should return something like:: Binary Modules Input ~~~~~~~~~~~~~~~~~~~~ -Support for binary modules was added in Ansible 2.1. When Ansible detects a binary module, it will proceed to +Support for binary modules was added in Ansible 2.2. When Ansible detects a binary module, it will proceed to supply the argument input as a file on ``argv[1]`` that is formatted as JSON. The JSON contents of that file would resemble something similar to:: From 3466e73c504b81d472809001a4d5c5df5f7f6172 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 12 Apr 2016 15:31:43 -0500 Subject: [PATCH 077/133] Resolve test failures --- test/units/plugins/action/test_action.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/units/plugins/action/test_action.py b/test/units/plugins/action/test_action.py index 2a9bb55693..62e4744cf8 100644 --- a/test/units/plugins/action/test_action.py +++ b/test/units/plugins/action/test_action.py @@ -217,7 +217,7 @@ class TestActionBase(unittest.TestCase): with patch.object(os, 'rename') as m: mock_task.args = dict(a=1, foo='fö〩') mock_connection.module_implementation_preferences = ('',) - (style, shebang, data) = action_base._configure_module(mock_task.action, mock_task.args) + (style, shebang, data, module_path, is_binary) = action_base._configure_module(mock_task.action, mock_task.args) self.assertEqual(style, "new") self.assertEqual(shebang, b"#!/usr/bin/python") @@ -229,7 +229,7 @@ class TestActionBase(unittest.TestCase): mock_task.action = 'win_copy' mock_task.args = dict(b=2) mock_connection.module_implementation_preferences = ('.ps1',) - (style, shebang, data) = action_base._configure_module('stat', mock_task.args) + (style, shebang, data, module_path, is_binary) = action_base._configure_module('stat', mock_task.args) self.assertEqual(style, "new") self.assertEqual(shebang, None) @@ -572,7 +572,7 @@ class TestActionBase(unittest.TestCase): action_base._low_level_execute_command = MagicMock() action_base._fixup_perms = MagicMock() - action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data') + action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', None, False) action_base._late_needs_tmp_path.return_value = False action_base._compute_environment_string.return_value = '' action_base._connection.has_pipelining = True @@ -581,12 +581,12 @@ class TestActionBase(unittest.TestCase): self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(rc=0, stdout="ok", stdout_lines=['ok'])) # test with needing/removing a remote tmp path - action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data') + action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', None, False) action_base._late_needs_tmp_path.return_value = True action_base._make_tmp_path.return_value = '/the/tmp/path' self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) - action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data') + action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data', None, False) self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) play_context.become = True @@ -594,14 +594,14 @@ class TestActionBase(unittest.TestCase): self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) # test an invalid shebang return - action_base._configure_module.return_value = ('new', '', 'this is the module data') + action_base._configure_module.return_value = ('new', '', 'this is the module data', None, False) action_base._late_needs_tmp_path.return_value = False self.assertRaises(AnsibleError, action_base._execute_module) # test with check mode enabled, once with support for check # mode and once with support disabled to raise an error play_context.check_mode = True - action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data') + action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', None, False) self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) action_base._supports_check_mode = False self.assertRaises(AnsibleError, action_base._execute_module) From 0faddfa1680ea2c3a1d2f6e02bd3139972a9b439 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 11 May 2016 15:14:01 -0500 Subject: [PATCH 078/133] Move binary module detection into executor/module_common.py --- lib/ansible/executor/module_common.py | 19 ++++++++++++---- lib/ansible/plugins/action/__init__.py | 29 ++++++++---------------- lib/ansible/plugins/action/async.py | 8 +++---- lib/ansible/plugins/shell/__init__.py | 1 + lib/ansible/plugins/shell/powershell.py | 2 +- test/units/plugins/action/test_action.py | 14 ++++++------ 6 files changed, 37 insertions(+), 36 deletions(-) diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index 631d67a750..ccfaab22ed 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -490,6 +490,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): # Save memory; the file won't have to be read again for this ansible module. del py_module_cache[py_module_file] +def _is_binary(module_path): + textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) + + with open(module_path, 'rb') as f: + start = f.read(1024) + return bool(start.translate(None, textchars)) + def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): """ Given the source of the module, convert it to a Jinja2 template to insert @@ -521,11 +528,13 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta module_substyle = 'jsonargs' elif b'WANT_JSON' in module_data: module_substyle = module_style = 'non_native_want_json' + elif _is_binary(module_path): + module_substyle = module_style = 'binary' shebang = None - # Neither old-style nor non_native_want_json modules should be modified + # Neither old-style, non_native_want_json nor binary modules should be modified # except for the shebang line (Done by modify_module) - if module_style in ('old', 'non_native_want_json'): + if module_style in ('old', 'non_native_want_json', 'binary'): return module_data, module_style, shebang output = BytesIO() @@ -731,7 +740,9 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) - if shebang is None: + if module_style == 'binary': + return (module_path, module_data, module_style, shebang) + elif shebang is None: lines = module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): shebang = lines[0].strip() @@ -753,4 +764,4 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul else: shebang = to_bytes(shebang, errors='strict') - return (module_data, module_style, shebang) + return (module_path, module_data, module_style, shebang) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 5e7113e867..6182368a40 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -100,13 +100,6 @@ class ActionBase(with_metaclass(ABCMeta, object)): return True return False - def _is_binary(self, module_path): - textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) - - with open(module_path, 'rb') as f: - start = f.read(1024) - return bool(start.translate(None, textchars)) - def _configure_module(self, module_name, module_args, task_vars=None): ''' Handles the loading and templating of the module code through the @@ -152,12 +145,9 @@ class ActionBase(with_metaclass(ABCMeta, object)): "run 'git submodule update --init --recursive' to correct this problem." % (module_name)) # insert shared code and arguments into the module - (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) + (module_path, module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) - if self._is_binary(module_path): - return ('non_native_want_json', None, None, module_path, True) - - return (module_style, module_shebang, module_data, module_path, False) + return (module_style, module_shebang, module_data, module_path) def _compute_environment_string(self): ''' @@ -301,7 +291,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): return remote_path - def _fixup_perms(self, remote_path, remote_user, execute=False, recursive=True): + def _fixup_perms(self, remote_path, remote_user, execute=True, recursive=True): """ We need the files we upload to be readable (and sometimes executable) by the user being sudo'd to but we want to limit other people's access @@ -579,9 +569,8 @@ class ActionBase(with_metaclass(ABCMeta, object)): # let module know our verbosity module_args['_ansible_verbosity'] = display.verbosity - (module_style, shebang, module_data, module_path, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) - - if not shebang and not is_binary: + (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + if not shebang and module_style != 'binary': raise AnsibleError("module (%s) is missing interpreter line" % module_name) # a remote tmp path may be necessary and not already created @@ -593,13 +582,13 @@ class ActionBase(with_metaclass(ABCMeta, object)): if tmp: remote_module_filename = self._connection._shell.get_remote_filename(module_path) remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename) - if module_style in ['old', 'non_native_want_json']: + if module_style in ('old', 'non_native_want_json', 'binary'): # we'll also need a temp file to hold our module arguments args_file_path = self._connection._shell.join_path(tmp, 'args') if remote_module_path or module_style != 'new': display.debug("transferring module to remote") - if is_binary: + if module_style == 'binary': self._transfer_file(module_path, remote_module_path) else: self._transfer_data(remote_module_path, module_data) @@ -610,7 +599,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): for k,v in iteritems(module_args): args_data += '%s="%s" ' % (k, pipes.quote(text_type(v))) self._transfer_data(args_file_path, args_data) - elif module_style == 'non_native_want_json': + elif module_style in ('non_native_want_json', 'binary'): self._transfer_data(args_file_path, json.dumps(module_args)) display.debug("done transferring module to remote") @@ -618,7 +607,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): # Fix permissions of the tmp path and tmp files. This should be # called after all files have been transferred. - self._fixup_perms(tmp, remote_user, recursive=True, execute=is_binary) + self._fixup_perms(tmp, remote_user, recursive=True) cmd = "" in_data = None diff --git a/lib/ansible/plugins/action/async.py b/lib/ansible/plugins/action/async.py index b2d9370e9e..9b59f64bba 100644 --- a/lib/ansible/plugins/action/async.py +++ b/lib/ansible/plugins/action/async.py @@ -54,18 +54,18 @@ class ActionModule(ActionBase): module_args['_ansible_no_log'] = True # configure, upload, and chmod the target module - (module_style, shebang, module_data, module_path, is_binary) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) - if is_binary: + (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) + if module_style == 'binary': self._transfer_file(module_path, remote_module_path) else: self._transfer_data(remote_module_path, module_data) # configure, upload, and chmod the async_wrapper module - (async_module_style, shebang, async_module_data, _, _) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) + (async_module_style, shebang, async_module_data, _) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) self._transfer_data(async_module_path, async_module_data) argsfile = None - if module_style == 'non_native_want_json': + if module_style in ('non_native_want_json', 'binary'): argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), json.dumps(module_args)) elif module_style == 'old': args_data = "" diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py index 0e951b7591..2a3145c9f8 100644 --- a/lib/ansible/plugins/shell/__init__.py +++ b/lib/ansible/plugins/shell/__init__.py @@ -165,6 +165,7 @@ class ShellBase(object): # don't quote the cmd if it's an empty string, because this will break pipelining mode if cmd.strip() != '': cmd = pipes.quote(cmd) + cmd_parts = [] if shebang: shebang = shebang.replace("#!", "").strip() diff --git a/lib/ansible/plugins/shell/powershell.py b/lib/ansible/plugins/shell/powershell.py index dfbae1b428..505b2e01da 100644 --- a/lib/ansible/plugins/shell/powershell.py +++ b/lib/ansible/plugins/shell/powershell.py @@ -54,8 +54,8 @@ class ShellModule(object): return path return '\'%s\'' % path - # powershell requires that script files end with .ps1 def get_remote_filename(self, pathname): + # powershell requires that script files end with .ps1 base_name = os.path.basename(pathname.strip()) name, ext = os.path.splitext(base_name.strip()) if ext.lower() not in ['.ps1', '.exe']: diff --git a/test/units/plugins/action/test_action.py b/test/units/plugins/action/test_action.py index 62e4744cf8..209d2a8d5b 100644 --- a/test/units/plugins/action/test_action.py +++ b/test/units/plugins/action/test_action.py @@ -217,7 +217,7 @@ class TestActionBase(unittest.TestCase): with patch.object(os, 'rename') as m: mock_task.args = dict(a=1, foo='fö〩') mock_connection.module_implementation_preferences = ('',) - (style, shebang, data, module_path, is_binary) = action_base._configure_module(mock_task.action, mock_task.args) + (style, shebang, data, path) = action_base._configure_module(mock_task.action, mock_task.args) self.assertEqual(style, "new") self.assertEqual(shebang, b"#!/usr/bin/python") @@ -229,7 +229,7 @@ class TestActionBase(unittest.TestCase): mock_task.action = 'win_copy' mock_task.args = dict(b=2) mock_connection.module_implementation_preferences = ('.ps1',) - (style, shebang, data, module_path, is_binary) = action_base._configure_module('stat', mock_task.args) + (style, shebang, data, path) = action_base._configure_module('stat', mock_task.args) self.assertEqual(style, "new") self.assertEqual(shebang, None) @@ -572,7 +572,7 @@ class TestActionBase(unittest.TestCase): action_base._low_level_execute_command = MagicMock() action_base._fixup_perms = MagicMock() - action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', None, False) + action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', 'path') action_base._late_needs_tmp_path.return_value = False action_base._compute_environment_string.return_value = '' action_base._connection.has_pipelining = True @@ -581,12 +581,12 @@ class TestActionBase(unittest.TestCase): self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(rc=0, stdout="ok", stdout_lines=['ok'])) # test with needing/removing a remote tmp path - action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', None, False) + action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path') action_base._late_needs_tmp_path.return_value = True action_base._make_tmp_path.return_value = '/the/tmp/path' self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) - action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data', None, False) + action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data', 'path') self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) play_context.become = True @@ -594,14 +594,14 @@ class TestActionBase(unittest.TestCase): self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) # test an invalid shebang return - action_base._configure_module.return_value = ('new', '', 'this is the module data', None, False) + action_base._configure_module.return_value = ('new', '', 'this is the module data', 'path') action_base._late_needs_tmp_path.return_value = False self.assertRaises(AnsibleError, action_base._execute_module) # test with check mode enabled, once with support for check # mode and once with support disabled to raise an error play_context.check_mode = True - action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', None, False) + action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', 'path') self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) action_base._supports_check_mode = False self.assertRaises(AnsibleError, action_base._execute_module) From 2e8146c52f911edcf83b92f7f6edb9f22d73ea13 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 11 May 2016 15:17:18 -0500 Subject: [PATCH 079/133] Improve documentation about the JSON args file --- docsite/rst/developing_modules.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index f1f5d4a2a2..1ec1b4ecdd 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -211,10 +211,11 @@ Binary Modules Input Support for binary modules was added in Ansible 2.2. When Ansible detects a binary module, it will proceed to supply the argument input as a file on ``argv[1]`` that is formatted as JSON. The JSON contents of that file -would resemble something similar to:: +would resemble something similar to the following payload for a module accepting the same arguments as the +``ping`` module:: { - "name": "Ansible", + "data": "pong", "_ansible_verbosity": 4, "_ansible_diff": false, "_ansible_debug": false, From 651b83d8be2865e1c4f9a2c091ee7981dccb7d23 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 11 May 2016 15:36:29 -0500 Subject: [PATCH 080/133] Run test_binary_modules --- test/integration/Makefile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/integration/Makefile b/test/integration/Makefile index 350dd43e98..d99271b8ed 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -23,7 +23,9 @@ VAULT_PASSWORD_FILE = vault-password CONSUL_RUNNING := $(shell python consul_running.py) EUID := $(shell id -u -r) -all: setup test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_connection non_destructive destructive includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts +UNAME := $(shell uname | tr '[:upper:]' '[:lower:]') + +all: setup test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_connection non_destructive destructive includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules test_test_infra: # ensure fail/assert work locally and can stop execution with non-zero exit code @@ -286,7 +288,15 @@ no_log: setup [ "$$(ansible-playbook no_log_local.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -vvvvv | awk --source 'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "6/0" ] test_binary_modules: - cd library && GOOS=linux GOARCH=amd64 go build -o helloworld_linux helloworld.go - cd library && GOOS=windows GOARCH=amd64 go build -o helloworld_win32nt.exe helloworld.go - cd library && GOOS=darwin GOARCH=amd64 go build -o helloworld_darwin helloworld.go - ansible-playbook test_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS) + mytmpdir=$(MYTMPDIR); \ + ls -al $$mytmpdir; \ + curl https://storage.googleapis.com/golang/go1.6.2.$(UNAME)-amd64.tar.gz | tar -xz -C $$mytmpdir; \ + [ $$? != 0 ] && wget -qO- https://storage.googleapis.com/golang/go1.6.2.$(UNAME)-amd64.tar.gz | tar -xz -C $$mytmpdir; \ + ls -al $$mytmpdir; \ + cd library; \ + GOROOT=$$mytmpdir/go GOOS=linux GOARCH=amd64 $$mytmpdir/go/bin/go build -o helloworld_linux helloworld.go; \ + GOROOT=$$mytmpdir/go GOOS=windows GOARCH=amd64 $$mytmpdir/go/bin/go build -o helloworld_win32nt.exe helloworld.go; \ + GOROOT=$$mytmpdir/go GOOS=darwin GOARCH=amd64 $$mytmpdir/go/bin/go build -o helloworld_darwin helloworld.go; \ + cd ..; \ + rm -rf $$mytmpdir; \ + ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS) From 34adb54734375fe25d415a252bee0afdcabe1cd7 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 12 May 2016 12:46:07 -0500 Subject: [PATCH 081/133] Make _is_binary use already read module_data, move _is_binary check to the top of the stack --- lib/ansible/executor/module_common.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index ccfaab22ed..d87860ac62 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -490,11 +490,9 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): # Save memory; the file won't have to be read again for this ansible module. del py_module_cache[py_module_file] -def _is_binary(module_path): +def _is_binary(module_data): textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) - - with open(module_path, 'rb') as f: - start = f.read(1024) + start = module_data[:1024] return bool(start.translate(None, textchars)) def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): @@ -511,7 +509,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # module_substyle is extra information that's useful internally. It tells # us what we have to look to substitute in the module files and whether # we're using module replacer or ziploader to format the module itself. - if REPLACER in module_data: + if _is_binary(module_data): + module_substyle = module_style = 'binary' + elif REPLACER in module_data: # Do REPLACER before from ansible.module_utils because we need make sure # we substitute "from ansible.module_utils basic" for REPLACER module_style = 'new' @@ -528,8 +528,6 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta module_substyle = 'jsonargs' elif b'WANT_JSON' in module_data: module_substyle = module_style = 'non_native_want_json' - elif _is_binary(module_path): - module_substyle = module_style = 'binary' shebang = None # Neither old-style, non_native_want_json nor binary modules should be modified From ca22783086d40cdd0f2fe8fcde1361f8a5212cef Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 12 May 2016 12:53:30 -0500 Subject: [PATCH 082/133] modify_module does not need to return module_path, as the calling code already has access to it --- lib/ansible/executor/module_common.py | 4 ++-- lib/ansible/plugins/action/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index d87860ac62..2f98e048be 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -739,7 +739,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) if module_style == 'binary': - return (module_path, module_data, module_style, shebang) + return (module_data, module_style, shebang) elif shebang is None: lines = module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): @@ -762,4 +762,4 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul else: shebang = to_bytes(shebang, errors='strict') - return (module_path, module_data, module_style, shebang) + return (module_data, module_style, shebang) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 6182368a40..9414a149fc 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -145,7 +145,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): "run 'git submodule update --init --recursive' to correct this problem." % (module_name)) # insert shared code and arguments into the module - (module_path, module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) + (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) return (module_style, module_shebang, module_data, module_path) From 049e0ba276b89730c0aeb0e80a30bb5ef2b56a34 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 12 May 2016 18:38:30 -0500 Subject: [PATCH 083/133] Add note to changelog for 2.2 about binary modules --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 791b5db55d..6131f022bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Ansible Changes By Release ## 2.2 TBD - ACTIVE DEVELOPMENT +###Major Changes: + +* Added support for binary modules + ####New Modules: - aws * ec2_customer_gateway From a404d0ffe2115e02981e0b92a49264f5b076bf9a Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 12 May 2016 17:01:25 -0700 Subject: [PATCH 084/133] Update submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 9dfed7c849..a7e45db9e6 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 9dfed7c84924ea61b3724b8df2c366ec4bc364cb +Subproject commit a7e45db9e6f1170ad7c07011c9ee9380007cdffb diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 2665acb257..cd03f10b9c 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 2665acb2578bdcb6edf224a5790b83cfb3bad694 +Subproject commit cd03f10b9ccb9f972a4cf84bc3e756870257da59 From 186337db28f1111575bde9b2978de00364964578 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 12 May 2016 20:30:05 -0700 Subject: [PATCH 085/133] Ship constants to the modules via internal module params rather than a secondary dict. --- lib/ansible/executor/module_common.py | 18 ++---------------- lib/ansible/module_utils/basic.py | 17 +++++++++++++---- lib/ansible/module_utils/rax.py | 2 +- lib/ansible/plugins/action/__init__.py | 10 ++++++++++ .../module_utils/basic/test__log_invocation.py | 1 - .../units/module_utils/basic/test_exit_json.py | 6 +++--- test/units/module_utils/basic/test_log.py | 8 ++++---- .../module_utils/basic/test_run_command.py | 2 +- .../units/module_utils/basic/test_safe_eval.py | 2 +- test/units/module_utils/test_basic.py | 13 ++++++------- .../module_utils/test_distribution_version.py | 2 +- 11 files changed, 42 insertions(+), 39 deletions(-) diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index 2f98e048be..3e45447d11 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -388,12 +388,6 @@ def _get_shebang(interpreter, task_vars, args=tuple()): return (shebang, interpreter) -def _get_facility(task_vars): - facility = C.DEFAULT_SYSLOG_FACILITY - if 'ansible_syslog_facility' in task_vars: - facility = task_vars['ansible_syslog_facility'] - return facility - def recursive_finder(name, data, py_module_names, py_module_cache, zf): """ Using ModuleDepFinder, make sure we have all of the module_utils files that @@ -539,15 +533,7 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta py_module_names = set() if module_substyle == 'python': - # ziploader for new-style python classes - constants = dict( - SELINUX_SPECIAL_FS=C.DEFAULT_SELINUX_SPECIAL_FS, - SYSLOG_FACILITY=_get_facility(task_vars), - ANSIBLE_VERSION=__version__, - ) - params = dict(ANSIBLE_MODULE_ARGS=module_args, - ANSIBLE_MODULE_CONSTANTS=constants, - ) + params = dict(ANSIBLE_MODULE_ARGS=module_args,) python_repred_params = to_bytes(repr(json.dumps(params)), errors='strict') try: @@ -697,7 +683,7 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # The main event -- substitute the JSON args string into the module module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) - facility = b'syslog.' + to_bytes(_get_facility(task_vars), errors='strict') + facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict') module_data = module_data.replace(b'syslog.LOG_USER', facility) return (module_data, module_style, shebang) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 6e20d54bee..14348adf40 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -558,7 +558,7 @@ class AnsibleModule(object): self.run_command_environ_update = {} self.aliases = {} - self._legal_inputs = ['_ansible_check_mode', '_ansible_no_log', '_ansible_debug', '_ansible_diff', '_ansible_verbosity'] + self._legal_inputs = ['_ansible_check_mode', '_ansible_no_log', '_ansible_debug', '_ansible_diff', '_ansible_verbosity', '_ansible_selinux_special_fs', '_ansible_version', '_ansible_syslog_facility'] if add_file_common_args: for k, v in FILE_COMMON_ARGUMENTS.items(): @@ -782,7 +782,7 @@ class AnsibleModule(object): (device, mount_point, fstype, options, rest) = line.split(' ', 4) if path_mount_point == mount_point: - for fs in self.constants['SELINUX_SPECIAL_FS']: + for fs in self._selinux_special_fs: if fs in fstype: special_context = self.selinux_context(path_mount_point) return (True, special_context) @@ -1175,6 +1175,7 @@ class AnsibleModule(object): return aliases_results def _check_arguments(self, check_invalid_arguments): + self._syslog_facility = 'LOG_USER' for (k,v) in list(self.params.items()): if k == '_ansible_check_mode' and v: @@ -1194,6 +1195,15 @@ class AnsibleModule(object): elif k == '_ansible_verbosity': self._verbosity = v + elif k == '_ansible_selinux_special_fs': + self._selinux_special_fs = v + + elif k == '_ansible_syslog_facility': + self._syslog_facility = v + + elif k == '_ansible_version': + self.ansible_version = v + elif check_invalid_arguments and k not in self._legal_inputs: self.fail_json(msg="unsupported parameter for module: %s" % k) @@ -1505,7 +1515,6 @@ class AnsibleModule(object): try: self.params = params['ANSIBLE_MODULE_ARGS'] - self.constants = params['ANSIBLE_MODULE_CONSTANTS'] except KeyError: # This helper used too early for fail_json to work. print('\n{"msg": "Error: Module unable to locate ANSIBLE_MODULE_ARGS and ANSIBLE_MODULE_CONSTANTS in json data from stdin. Unable to figure out what parameters were passed", "failed": true}') @@ -1514,7 +1523,7 @@ class AnsibleModule(object): def _log_to_syslog(self, msg): if HAS_SYSLOG: module = 'ansible-%s' % os.path.basename(__file__) - facility = getattr(syslog, self.constants.get('SYSLOG_FACILITY', 'LOG_USER'), syslog.LOG_USER) + facility = getattr(syslog, self._syslog_facility, syslog.LOG_USER) syslog.openlog(str(module), 0, facility) syslog.syslog(syslog.LOG_INFO, msg) diff --git a/lib/ansible/module_utils/rax.py b/lib/ansible/module_utils/rax.py index 37ce66b40d..ad7afeb3ec 100644 --- a/lib/ansible/module_utils/rax.py +++ b/lib/ansible/module_utils/rax.py @@ -263,7 +263,7 @@ def rax_required_together(): def setup_rax_module(module, rax_module, region_required=True): """Set up pyrax in a standard way for all modules""" - rax_module.USER_AGENT = 'ansible/%s %s' % (module.constants['ANSIBLE_VERSION'], + rax_module.USER_AGENT = 'ansible/%s %s' % (module.ansible_version, rax_module.USER_AGENT) api_key = module.params.get('api_key') diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 60dc199204..1d2275ae61 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -35,6 +35,7 @@ from ansible.compat.six import binary_type, text_type, iteritems, with_metaclass from ansible import constants as C from ansible.errors import AnsibleError, AnsibleConnectionFailure from ansible.executor.module_common import modify_module +from ansible.release import __version__ from ansible.parsing.utils.jsonify import jsonify from ansible.utils.unicode import to_bytes, to_unicode @@ -570,6 +571,15 @@ class ActionBase(with_metaclass(ABCMeta, object)): # let module know our verbosity module_args['_ansible_verbosity'] = display.verbosity + # give the module information about the ansible version + module_args['_ansible_version'] = __version__ + + # set the syslog facility to be used in the module + module_args['_ansible_syslog_facility'] = task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY) + + # let module know about filesystems that selinux treats specially + module_args['_ansible_selinux_special_fs'] = C.DEFAULT_SELINUX_SPECIAL_FS + (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) if not shebang and module_style != 'binary': raise AnsibleError("module (%s) is missing interpreter line" % module_name) diff --git a/test/units/module_utils/basic/test__log_invocation.py b/test/units/module_utils/basic/test__log_invocation.py index 677eaa2c90..d4510c5efc 100644 --- a/test/units/module_utils/basic/test__log_invocation.py +++ b/test/units/module_utils/basic/test__log_invocation.py @@ -36,7 +36,6 @@ class TestModuleUtilsBasic(unittest.TestCase): dict( ANSIBLE_MODULE_ARGS=dict( foo=False, bar=[1,2,3], bam="bam", baz=u'baz'), - ANSIBLE_MODULE_CONSTANTS=dict() ))): from ansible.module_utils import basic diff --git a/test/units/module_utils/basic/test_exit_json.py b/test/units/module_utils/basic/test_exit_json.py index 2cc6239845..e92f328ef2 100644 --- a/test/units/module_utils/basic/test_exit_json.py +++ b/test/units/module_utils/basic/test_exit_json.py @@ -35,7 +35,7 @@ empty_invocation = {u'module_args': {}} @unittest.skipIf(sys.version_info[0] >= 3, "Python 3 is not supported on targets (yet)") class TestAnsibleModuleExitJson(unittest.TestCase): def setUp(self): - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) self.stdin_swap_ctx = swap_stdin_and_argv(stdin_data=args) self.stdin_swap_ctx.__enter__() @@ -120,7 +120,7 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase): def test_exit_json_removes_values(self): self.maxDiff = None for args, return_val, expected in self.dataset: - params = dict(ANSIBLE_MODULE_ARGS=args, ANSIBLE_MODULE_CONSTANTS={}) + params = dict(ANSIBLE_MODULE_ARGS=args) params = json.dumps(params) with swap_stdin_and_argv(stdin_data=params): @@ -143,7 +143,7 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase): expected = copy.deepcopy(expected) del expected['changed'] expected['failed'] = True - params = dict(ANSIBLE_MODULE_ARGS=args, ANSIBLE_MODULE_CONSTANTS={}) + params = dict(ANSIBLE_MODULE_ARGS=args) params = json.dumps(params) with swap_stdin_and_argv(stdin_data=params): with swap_stdout(): diff --git a/test/units/module_utils/basic/test_log.py b/test/units/module_utils/basic/test_log.py index 6e47e46c00..c162504ca5 100644 --- a/test/units/module_utils/basic/test_log.py +++ b/test/units/module_utils/basic/test_log.py @@ -43,7 +43,7 @@ except ImportError: class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase): def setUp(self): - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap = swap_stdin_and_argv(stdin_data=args) @@ -80,7 +80,7 @@ class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase): class TestAnsibleModuleJournaldSmokeTest(unittest.TestCase): def setUp(self): - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap = swap_stdin_and_argv(stdin_data=args) @@ -129,7 +129,7 @@ class TestAnsibleModuleLogSyslog(unittest.TestCase): } def setUp(self): - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() @@ -190,7 +190,7 @@ class TestAnsibleModuleLogJournal(unittest.TestCase): # overriding run lets us use context managers for setup/teardown-esque behavior def setUp(self): - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() diff --git a/test/units/module_utils/basic/test_run_command.py b/test/units/module_utils/basic/test_run_command.py index a78825d46b..35186e220e 100644 --- a/test/units/module_utils/basic/test_run_command.py +++ b/test/units/module_utils/basic/test_run_command.py @@ -62,7 +62,7 @@ class TestAnsibleModuleRunCommand(unittest.TestCase): if path == '/inaccessible': raise OSError(errno.EPERM, "Permission denied: '/inaccessible'") - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() diff --git a/test/units/module_utils/basic/test_safe_eval.py b/test/units/module_utils/basic/test_safe_eval.py index cc2aaf8a53..393416c0fd 100644 --- a/test/units/module_utils/basic/test_safe_eval.py +++ b/test/units/module_utils/basic/test_safe_eval.py @@ -31,7 +31,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase): def test_module_utils_basic_safe_eval(self): from ansible.module_utils import basic - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) with swap_stdin_and_argv(stdin_data=args): basic._ANSIBLE_ARGS = None diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index 42825ff304..d2025235bc 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -41,7 +41,7 @@ realimport = builtins.__import__ class TestModuleUtilsBasic(unittest.TestCase): def setUp(self): - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) # unittest doesn't have a clean place to use a context manager, so we have to enter/exit manually self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() @@ -288,7 +288,7 @@ class TestModuleUtilsBasic(unittest.TestCase): req_to = (('bam', 'baz'),) # should test ok - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo": "hello"}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo": "hello"})) with swap_stdin_and_argv(stdin_data=args): basic._ANSIBLE_ARGS = None @@ -305,7 +305,7 @@ class TestModuleUtilsBasic(unittest.TestCase): # FIXME: add asserts here to verify the basic config # fail, because a required param was not specified - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) with swap_stdin_and_argv(stdin_data=args): basic._ANSIBLE_ARGS = None @@ -322,7 +322,7 @@ class TestModuleUtilsBasic(unittest.TestCase): ) # fail because of mutually exclusive parameters - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo":"hello", "bar": "bad", "bam": "bad"}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"foo":"hello", "bar": "bad", "bam": "bad"})) with swap_stdin_and_argv(stdin_data=args): self.assertRaises( @@ -338,7 +338,7 @@ class TestModuleUtilsBasic(unittest.TestCase): ) # fail because a param required due to another param was not specified - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"bam": "bad"}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={"bam": "bad"})) with swap_stdin_and_argv(stdin_data=args): self.assertRaises( @@ -550,14 +550,13 @@ class TestModuleUtilsBasic(unittest.TestCase): from ansible.module_utils import basic basic._ANSIBLE_ARGS = None - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={"SELINUX_SPECIAL_FS": "nfs,nfsd,foos"})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={'_ansible_selinux_special_fs': "nfs,nfsd,foos"})) with swap_stdin_and_argv(stdin_data=args): am = basic.AnsibleModule( argument_spec = dict(), ) - print(am.constants) def _mock_find_mount_point(path): if path.startswith('/some/path'): diff --git a/test/units/module_utils/test_distribution_version.py b/test/units/module_utils/test_distribution_version.py index 3ade3b0c3a..1f9a767f03 100644 --- a/test/units/module_utils/test_distribution_version.py +++ b/test/units/module_utils/test_distribution_version.py @@ -400,7 +400,7 @@ def test_distribution_version(): from ansible.module_utils import basic - args = json.dumps(dict(ANSIBLE_MODULE_ARGS={}, ANSIBLE_MODULE_CONSTANTS={})) + args = json.dumps(dict(ANSIBLE_MODULE_ARGS={})) with swap_stdin_and_argv(stdin_data=args): module = basic.AnsibleModule(argument_spec=dict()) From d2bade6dafee19d7d010e8f18744f02409864980 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Fri, 13 May 2016 10:00:23 -0400 Subject: [PATCH 086/133] Make sure setting facts with run_once makes copies of the data When using run_once, there is only one dict of facts so passing that to the VariableManager results in the fact cache containing the same dictionary reference for all hosts in inventory. This patch fixes that by making sure we pass a copy of the facts dict to VariableManager. Fixes #14279 --- lib/ansible/plugins/strategy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index 25b2791577..c93626a7ed 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -377,9 +377,9 @@ class StrategyBase: facts = result[4] for target_host in host_list: if task.action == 'set_fact': - self._variable_manager.set_nonpersistent_facts(target_host, facts) + self._variable_manager.set_nonpersistent_facts(target_host, facts.copy()) else: - self._variable_manager.set_host_facts(target_host, facts) + self._variable_manager.set_host_facts(target_host, facts.copy()) elif result[0].startswith('v2_runner_item') or result[0] == 'v2_runner_retry': self._tqm.send_callback(result[0], result[1]) elif result[0] == 'v2_on_file_diff': From 5b7896ce7fff69c58e0ba8c22d813a00c53e21c6 Mon Sep 17 00:00:00 2001 From: Jason McKerr Date: Fri, 13 May 2016 10:43:46 -0400 Subject: [PATCH 087/133] Update committer_guidelines.rst Adding Ryan and Adrian --- docsite/rst/committer_guidelines.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docsite/rst/committer_guidelines.rst b/docsite/rst/committer_guidelines.rst index 5dc0af6d47..4ce077c2f1 100644 --- a/docsite/rst/committer_guidelines.rst +++ b/docsite/rst/committer_guidelines.rst @@ -83,3 +83,5 @@ Individuals who've been asked to become a part of this group have generally been * Trond Hindenes * Jon Hawkesworth * Will Thames +* Adrian Likins +* Ryan Brown From 878b0dca6826fdfabf5c639d3252cdcc738e4546 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 13 May 2016 09:44:00 -0500 Subject: [PATCH 088/133] Use .code instead of .getcode() as py24 does not have .getcode(). Fixes https://github.com/ansible/ansible-modules-core/issues/3608 --- lib/ansible/module_utils/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index ec8484a52f..01da3c038b 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -893,7 +893,7 @@ def fetch_url(module, url, data=None, headers=None, method=None, url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth, follow_redirects=follow_redirects) info.update(r.info()) - info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), url=r.geturl(), status=r.getcode())) + info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), url=r.geturl(), status=r.code)) except NoSSLError: e = get_exception() distribution = get_distribution() From 119baba6b1771ebca4d0ab1c5544a4f8af8b3cb9 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 13 May 2016 10:17:20 -0700 Subject: [PATCH 089/133] Update submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index a7e45db9e6..83a3c45a7d 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit a7e45db9e6f1170ad7c07011c9ee9380007cdffb +Subproject commit 83a3c45a7d0cf369fa7fcb0905db3b19d752aa79 diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index cd03f10b9c..148a0ddabf 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit cd03f10b9ccb9f972a4cf84bc3e756870257da59 +Subproject commit 148a0ddabf0bf53a4fed782969212b5c22e3ed04 From e083fa3d1193c886002d286a0b2d210c9657a092 Mon Sep 17 00:00:00 2001 From: jctanner Date: Fri, 13 May 2016 13:39:04 -0400 Subject: [PATCH 090/133] Disable sftp batch mode if sshpass (#15829) Make use of the -oBatchMode=no option to force password prompts from sftp Addresses #13401 --- lib/ansible/plugins/connection/ssh.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index b03b15fc36..06dca813e3 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -133,8 +133,12 @@ class Connection(ConnectionBase): ## Next, additional arguments based on the configuration. # sftp batch mode allows us to correctly catch failed transfers, but can - # be disabled if the client side doesn't support the option. + # be disabled if the client side doesn't support the option. However, + # sftp batch mode does not prompt for passwords so it must be disabled + # if not using controlpersist and using sshpass if binary == 'sftp' and C.DEFAULT_SFTP_BATCH_MODE: + if self._play_context.password: + self._add_args('disable batch mode for sshpass', ['-o', 'BatchMode=no']) self._command += ['-b', '-'] self._command += ['-C'] From 03d33f09052f9440945d5f8dfa16821127c0bb31 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 13 May 2016 11:01:52 -0700 Subject: [PATCH 091/133] Start adding required to docs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 83a3c45a7d..298fd0ae56 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 83a3c45a7d0cf369fa7fcb0905db3b19d752aa79 +Subproject commit 298fd0ae56713e851fda06da13c5662f6be549dc diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 148a0ddabf..f953d5dc0c 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 148a0ddabf0bf53a4fed782969212b5c22e3ed04 +Subproject commit f953d5dc0c1418dcdc5db72e8fb630b6aa83997f From 849cdd90f2cf69bb61e0152bbd06772147cbc5dc Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Fri, 13 May 2016 12:01:05 -0700 Subject: [PATCH 092/133] update 2.1 completed roadmap items --- ROADMAP.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 9c205b1483..77f4067645 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -17,13 +17,13 @@ These roadmaps are the team's *best guess* roadmaps based on the Ansible team's ## Windows, General * Figuring out privilege escalation (runas w/ username/password) * Implement kerberos encryption over http -* pywinrm conversion to requests (Some mess here on pywinrm/requests. will need docs etc.) -* NTLM support +* ~~pywinrm conversion to requests (Some mess here on pywinrm/requests. will need docs etc.)~~ DONE +* ~~NTLM support~~ DONE ## Modules * Windows - * Finish cleaning up tests and support for post-beta release - * Strict mode cleanup (one module in core) + * ~~Finish cleaning up tests and support for post-beta release~~ DONE + * ~~Strict mode cleanup (one module in core)~~ DONE * Domain user/group management * Finish win\_host and win\_rm in the domain/workgroup modules. * Close 2 existing PRs (These were deemed insufficient) @@ -42,16 +42,16 @@ These roadmaps are the team's *best guess* roadmaps based on the Ansible team's * VMware modules moved to official pyvmomi bindings * VMware inventory script updates for pyvmomi, adding tagging support * Azure (Notes: We've made progress here now that Microsoft has swaped out the code generator on the Azure Python SDK. We have basic modules working against all of these resources at this time. Could ship it against current SDK, but may break. Or should the version be pinned?) - * Minimal Azure coverage using new ARM api - * Resource Group - * Virtual Network - * Subnet - * Public IP - * Network Interface - * Storage Account - * Security Group - * Virtual Machine - * Update of inventory script to use new API, adding tagging support + * ~~Minimal Azure coverage using new ARM api~~ DONE + * ~~Resource Group~~ DONE + * ~~Virtual Network~~ DONE + * ~~Subnet~~ DONE + * ~~Public IP~~ DONE + * ~~Network Interface~~ DONE + * ~~Storage Account~~ DONE + * ~~Security Group~~ DONE + * ~~Virtual Machine~~ DONE + * ~~Update of inventory script to use new API, adding tagging support~~ DONE * Docker: * Start Docker module refactor * Update to match current docker CLI capabilities From 145f2df1f69382de3359900eed0ef0a714594884 Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Fri, 13 May 2016 12:26:43 -0700 Subject: [PATCH 093/133] updated committer IRC nicks/GH IDs --- docsite/rst/committer_guidelines.rst | 50 ++++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docsite/rst/committer_guidelines.rst b/docsite/rst/committer_guidelines.rst index 4ce077c2f1..592beeb62d 100644 --- a/docsite/rst/committer_guidelines.rst +++ b/docsite/rst/committer_guidelines.rst @@ -60,28 +60,28 @@ People ====== Individuals who've been asked to become a part of this group have generally been contributing in significant ways to the Ansible community for some time. Should they agree, they are requested to add their names and GitHub IDs to this file, in the section below, via a pull request. Doing so indicates that these individuals agree to act in the ways that their fellow committers trust that they will act. -* James Cammarata -* Brian Coca -* Matt Davis -* Toshio Kuratomi -* Jason McKerr -* Robyn Bergeron -* Greg DeKoenigsberg -* Monty Taylor -* Matt Martz -* Nate Case -* James Tanner -* Peter Sprygada -* Abhijit Menon-Sen -* Michael Scherer -* RenĂ© Moser -* David Shrewsbury -* Sandra Wills -* Graham Mainwaring -* Jon Davila -* Chris Houseknecht -* Trond Hindenes -* Jon Hawkesworth -* Will Thames -* Adrian Likins -* Ryan Brown +* James Cammarata (IRC: jimi|ansible, GitHub: @jimi-c) +* Brian Coca (IRC: bcoca, GitHub: @bcoca) +* Matt Davis (IRC: nitzmahone, GitHub: @nitzmahone) +* Toshio Kuratomi (IRC: abadger1999, GitHub: @abadger) +* Jason McKerr (IRC: newtMcKerr, GitHub: @mckerrj) +* Robyn Bergeron (IRC: rbergeron, GitHub: @robynbergeron) +* Greg DeKoenigsberg (IRC: gregdek, GitHub: @gregdek) +* Monty Taylor (GitHub: @emonty) +* Matt Martz (IRC: sivel, GitHub: @sivel) +* Nate Case (IRC: Qualthos, GitHub: @Qalthos) +* James Tanner (IRC: jtanner, GitHub: @jctanner) +* Peter Sprygada (GitHub: @privateip) +* Abhijit Menon-Sen (GitHub: @amenonsen) +* Michael Scherer (GitHub: @mscherer) +* RenĂ© Moser (GitHub: @resmo) +* David Shrewsbury (IRC: Shrews, GitHub: @Shrews) +* Sandra Wills (IRC: docschick, GitHub: @docschick) +* Graham Mainwaring (GitHub: @ghjm) +* Jon Davila (GitHub: @defionscode) +* Chris Houseknecht (GitHub: @chouseknecht) +* Trond Hindenes (GitHub: @trondhindenes) +* Jon Hawkesworth (GitHub: @jhawkesworth) +* Will Thames (IRC: willthames, GitHub: @willthames) +* Adrian Likins (IRC: alikins, GitHub: @alikins) +* Ryan Brown (IRC: ryansb, GitHub: @ryansb) From 6d4ba4a161ff78a3b97117eab15aebaa3c5317dd Mon Sep 17 00:00:00 2001 From: Nathaniel Case Date: Fri, 13 May 2016 16:02:45 -0400 Subject: [PATCH 094/133] Fix my IRC handle Not the first nor last time someone has gotten this wrong, but oddly only one ofthem was wrong --- docsite/rst/committer_guidelines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/committer_guidelines.rst b/docsite/rst/committer_guidelines.rst index 592beeb62d..53a7ef14b0 100644 --- a/docsite/rst/committer_guidelines.rst +++ b/docsite/rst/committer_guidelines.rst @@ -69,7 +69,7 @@ Individuals who've been asked to become a part of this group have generally been * Greg DeKoenigsberg (IRC: gregdek, GitHub: @gregdek) * Monty Taylor (GitHub: @emonty) * Matt Martz (IRC: sivel, GitHub: @sivel) -* Nate Case (IRC: Qualthos, GitHub: @Qalthos) +* Nate Case (IRC: Qalthos, GitHub: @Qalthos) * James Tanner (IRC: jtanner, GitHub: @jctanner) * Peter Sprygada (GitHub: @privateip) * Abhijit Menon-Sen (GitHub: @amenonsen) From 4f0be29d6594f5aa52e6b4c4bb520f2a696ff0a0 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Thu, 12 May 2016 14:42:24 -0400 Subject: [PATCH 095/133] Reworking retry/until logic to fix bugs Prior to this patch, the retry/until logic would fail any task that succeeded if it took all of the alloted retries to succeed. This patch reworks the retry/until logic to make things more simple and clear. Fixes #15697 --- lib/ansible/executor/task_executor.py | 24 ++++++++++++------------ lib/ansible/playbook/task.py | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 09ee933eb2..5f172da0d3 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -419,10 +419,10 @@ class TaskExecutor: self._task.args = dict((i[0], i[1]) for i in iteritems(self._task.args) if i[1] != omit_token) # Read some values from the task, so that we can modify them if need be - if self._task.until is not None: + if self._task.until: retries = self._task.retries - if retries <= 0: - retries = 1 + if retries is None: + retries = 3 else: retries = 1 @@ -436,7 +436,7 @@ class TaskExecutor: display.debug("starting attempt loop") result = None - for attempt in range(retries): + for attempt in range(1, retries + 1): display.debug("running the handler") try: result = self._handler.run(task_vars=variables) @@ -499,23 +499,23 @@ class TaskExecutor: _evaluate_changed_when_result(result) _evaluate_failed_when_result(result) - if attempt < retries - 1: + if retries > 1: cond = Conditional(loader=self._loader) cond.when = self._task.until if cond.evaluate_conditional(templar, vars_copy): break else: # no conditional check, or it failed, so sleep for the specified time - result['attempts'] = attempt + 1 - result['retries'] = retries - result['_ansible_retry'] = True - display.debug('Retrying task, attempt %d of %d' % (attempt + 1, retries)) - self._rslt_q.put(TaskResult(self._host, self._task, result), block=False) - time.sleep(delay) + if attempt < retries: + result['attempts'] = attempt + result['_ansible_retry'] = True + result['retries'] = retries + display.debug('Retrying task, attempt %d of %d' % (attempt, retries)) + self._rslt_q.put(TaskResult(self._host, self._task, result), block=False) + time.sleep(delay) else: if retries > 1: # we ran out of attempts, so mark the result as failed - result['attempts'] = retries result['failed'] = True # do the final update of the local variables here, for both registered diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index 54bfdc960b..c16d860985 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -84,7 +84,7 @@ class Task(Base, Conditional, Taggable, Become): _notify = FieldAttribute(isa='list') _poll = FieldAttribute(isa='int') _register = FieldAttribute(isa='string') - _retries = FieldAttribute(isa='int', default=3) + _retries = FieldAttribute(isa='int') _until = FieldAttribute(isa='list', default=[]) def __init__(self, block=None, role=None, task_include=None): From 043e9106525610e637639f8a4a18a9a98942e59a Mon Sep 17 00:00:00 2001 From: Nathaniel Case Date: Fri, 13 May 2016 16:26:07 -0400 Subject: [PATCH 096/133] Check for jxmlease when using netconf on JUNOS. (#15835) --- lib/ansible/module_utils/junos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ansible/module_utils/junos.py b/lib/ansible/module_utils/junos.py index 61d5ca73ce..bf1a2f955d 100644 --- a/lib/ansible/module_utils/junos.py +++ b/lib/ansible/module_utils/junos.py @@ -350,6 +350,8 @@ def get_module(**kwargs): module.fail_json(msg='paramiko is required but does not appear to be installed') elif module.params['transport'] == 'netconf' and not HAS_PYEZ: module.fail_json(msg='junos-eznc >= 1.2.2 is required but does not appear to be installed') + elif module.params['transport'] == 'netconf' and not HAS_JXMLEASE: + module.fail_json(msg='jxmlease is required but does not appear to be installed') module.connect() return module From a2c905c32ef86b014552b8fcc87c21edda5cc75e Mon Sep 17 00:00:00 2001 From: Thomas Quinot Date: Fri, 13 May 2016 22:57:17 +0200 Subject: [PATCH 097/133] Fix uninitialized distribution fact on FreeBSD (#15842) Initialize facts['distribution'] with self.system so that this fact does not remain uninitialized on systems_platform_working platforms (FreeBSD, OpenBSD). Fixes #15841 --- lib/ansible/module_utils/facts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index fa23d059b0..03121e6814 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -660,6 +660,7 @@ class Distribution(object): # The platform module provides information about the running # system/distribution. Use this as a baseline and fix buggy systems # afterwards + self.facts['distribution'] = self.system self.facts['distribution_release'] = platform.release() self.facts['distribution_version'] = platform.version() From 6f6456dff546d98806967164808bfdd9124357f1 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Fri, 13 May 2016 16:59:36 -0400 Subject: [PATCH 098/133] Adding a deprecation message for accelerated mode --- lib/ansible/executor/task_executor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 5f172da0d3..04ea22a02e 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -634,6 +634,8 @@ class TaskExecutor: raise AnsibleError("the connection plugin '%s' was not found" % conn_type) if self._play_context.accelerate: + # accelerate is deprecated as of 2.1... + display.deprecated('Accelerated mode is deprecated. Consider using SSH with ControlPersist and pipelining enabled instead') # launch the accelerated daemon here ssh_connection = connection handler = self._shared_loader_obj.action_loader.get( From 2ffe30a8c2d318a1e1188916e23c9ccae58bef27 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 13 May 2016 17:03:37 -0400 Subject: [PATCH 099/133] converted list to table and added some aliases --- docsite/rst/committer_guidelines.rst | 79 +++++++++++++++++++--------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/docsite/rst/committer_guidelines.rst b/docsite/rst/committer_guidelines.rst index 53a7ef14b0..4b59bbff9a 100644 --- a/docsite/rst/committer_guidelines.rst +++ b/docsite/rst/committer_guidelines.rst @@ -60,28 +60,57 @@ People ====== Individuals who've been asked to become a part of this group have generally been contributing in significant ways to the Ansible community for some time. Should they agree, they are requested to add their names and GitHub IDs to this file, in the section below, via a pull request. Doing so indicates that these individuals agree to act in the ways that their fellow committers trust that they will act. -* James Cammarata (IRC: jimi|ansible, GitHub: @jimi-c) -* Brian Coca (IRC: bcoca, GitHub: @bcoca) -* Matt Davis (IRC: nitzmahone, GitHub: @nitzmahone) -* Toshio Kuratomi (IRC: abadger1999, GitHub: @abadger) -* Jason McKerr (IRC: newtMcKerr, GitHub: @mckerrj) -* Robyn Bergeron (IRC: rbergeron, GitHub: @robynbergeron) -* Greg DeKoenigsberg (IRC: gregdek, GitHub: @gregdek) -* Monty Taylor (GitHub: @emonty) -* Matt Martz (IRC: sivel, GitHub: @sivel) -* Nate Case (IRC: Qalthos, GitHub: @Qalthos) -* James Tanner (IRC: jtanner, GitHub: @jctanner) -* Peter Sprygada (GitHub: @privateip) -* Abhijit Menon-Sen (GitHub: @amenonsen) -* Michael Scherer (GitHub: @mscherer) -* RenĂ© Moser (GitHub: @resmo) -* David Shrewsbury (IRC: Shrews, GitHub: @Shrews) -* Sandra Wills (IRC: docschick, GitHub: @docschick) -* Graham Mainwaring (GitHub: @ghjm) -* Jon Davila (GitHub: @defionscode) -* Chris Houseknecht (GitHub: @chouseknecht) -* Trond Hindenes (GitHub: @trondhindenes) -* Jon Hawkesworth (GitHub: @jhawkesworth) -* Will Thames (IRC: willthames, GitHub: @willthames) -* Adrian Likins (IRC: alikins, GitHub: @alikins) -* Ryan Brown (IRC: ryansb, GitHub: @ryansb) ++---------------------+----------------------+--------------------+----------------------+ +| Name | Github ID | IRC Nick | Other | ++=====================+======================+====================+======================+ ++---------------------+----------------------+--------------------+----------------------+ +| James Cammarata | jimi-c | jimi | | ++---------------------+----------------------+--------------------+----------------------+ +| Brian Coca | bcoca | bcoca | mdyson@cyberdyne.com | ++---------------------+----------------------+--------------------+----------------------+ +| Matt Davis | nitzmahone | nitzmahone | | ++---------------------+----------------------+--------------------+----------------------+ +| Toshio Kuratomi | abadger | abadger1999 | | ++---------------------+----------------------+--------------------+----------------------+ +| Jason McKerr | mckerrj | newtMcKerr | | ++---------------------+----------------------+--------------------+----------------------+ +| Robyn Bergeron | robynbergeron | rbergeron | | ++---------------------+----------------------+--------------------+----------------------+ +| Greg DeKoenigsberg | gregdek | gregdek | | ++---------------------+----------------------+--------------------+----------------------+ +| Monty Taylor | emonty | mordred | | ++---------------------+----------------------+--------------------+----------------------+ +| Matt Martz | sivel | sivel | | ++---------------------+----------------------+--------------------+----------------------+ +| Nate Case | qalthos | Qalthos | | ++---------------------+----------------------+--------------------+----------------------+ +| James Tanner | jctanner | jtanner | | ++---------------------+----------------------+--------------------+----------------------+ +| Peter Sprygada | privateip | privateip | | ++---------------------+----------------------+--------------------+----------------------+ +| Abhijit Menon-Sen | amenonsen | crab | | ++---------------------+----------------------+--------------------+----------------------+ +| Michael Scherer | mscherer | | | ++---------------------+----------------------+--------------------+----------------------+ +| RenĂ© Moser | resmo | resmo | | ++---------------------+----------------------+--------------------+----------------------+ +| David Shrewsbury | Shrews | Shrews | | ++---------------------+----------------------+--------------------+----------------------+ +| Sandra Wills | docschick | docschick | | ++---------------------+----------------------+--------------------+----------------------+ +| Graham Mainwaring | ghjm | | | ++---------------------+----------------------+--------------------+----------------------+ +| Jon Davila | defionscode | | | ++---------------------+----------------------+--------------------+----------------------+ +| Chris Houseknecht | chouseknecht | | | ++---------------------+----------------------+--------------------+----------------------+ +| Trond Hindenes | trondhindenes | | | ++---------------------+----------------------+--------------------+----------------------+ +| Jon Hawkesworth | jhawkseworth | jhawkseworth | | ++---------------------+----------------------+--------------------+----------------------+ +| Will Thames | wilthames | willthames | | ++---------------------+----------------------+--------------------+----------------------+ +| Ryan Brown | ryansb | ryansb | | ++---------------------+----------------------+--------------------+----------------------+ +| Adrian Likins | alikins | alikins | | ++---------------------+----------------------+--------------------+----------------------+ From acf3c4dedd732338a303b6614b1bf089a7980b99 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 13 May 2016 17:16:11 -0400 Subject: [PATCH 100/133] removed emepty line --- docsite/rst/committer_guidelines.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docsite/rst/committer_guidelines.rst b/docsite/rst/committer_guidelines.rst index 4b59bbff9a..6ca974eeba 100644 --- a/docsite/rst/committer_guidelines.rst +++ b/docsite/rst/committer_guidelines.rst @@ -63,7 +63,6 @@ Individuals who've been asked to become a part of this group have generally been +---------------------+----------------------+--------------------+----------------------+ | Name | Github ID | IRC Nick | Other | +=====================+======================+====================+======================+ -+---------------------+----------------------+--------------------+----------------------+ | James Cammarata | jimi-c | jimi | | +---------------------+----------------------+--------------------+----------------------+ | Brian Coca | bcoca | bcoca | mdyson@cyberdyne.com | From 08f71c400fb15f56ea5619f45a029bea196a7521 Mon Sep 17 00:00:00 2001 From: Peter Schaadt Date: Fri, 13 May 2016 14:47:08 -0700 Subject: [PATCH 101/133] Fixing typo in Inventory documentation. (#15858) --- docsite/rst/intro_inventory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/intro_inventory.rst b/docsite/rst/intro_inventory.rst index d64eb130eb..f10dbd8a03 100644 --- a/docsite/rst/intro_inventory.rst +++ b/docsite/rst/intro_inventory.rst @@ -300,7 +300,7 @@ ansible_become ansible_docker_extra_args Could be a string with any additional arguments understood by Docker, which are not command specific. This parameter is mainly used to configure a remote Docker daemon to use. -Here an example of how to instantly depoloy to created containers:: +Here is an example of how to instantly deploy to created containers:: - name: create jenkins container docker: From 85477fa215635201f6dbc92cfb344b874c31423f Mon Sep 17 00:00:00 2001 From: Robin Roth Date: Sat, 14 May 2016 06:09:13 +0200 Subject: [PATCH 102/133] Don't use 'from ansible.module_utils import foo' style here as it breaks (#15756) py.test" --- test/units/module_utils/basic/test_run_command.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/units/module_utils/basic/test_run_command.py b/test/units/module_utils/basic/test_run_command.py index 35186e220e..0ba6f4cd16 100644 --- a/test/units/module_utils/basic/test_run_command.py +++ b/test/units/module_utils/basic/test_run_command.py @@ -30,8 +30,7 @@ from ansible.compat.tests.mock import call, MagicMock, Mock, patch, sentinel from units.mock.procenv import swap_stdin_and_argv -from ansible.module_utils import basic -from ansible.module_utils.basic import AnsibleModule +import ansible.module_utils.basic class OpenBytesIO(BytesIO): """BytesIO with dummy close() method @@ -67,8 +66,8 @@ class TestAnsibleModuleRunCommand(unittest.TestCase): self.stdin_swap = swap_stdin_and_argv(stdin_data=args) self.stdin_swap.__enter__() - basic._ANSIBLE_ARGS = None - self.module = AnsibleModule(argument_spec=dict()) + ansible.module_utils.basic._ANSIBLE_ARGS = None + self.module = ansible.module_utils.basic.AnsibleModule(argument_spec=dict()) self.module.fail_json = MagicMock(side_effect=SystemExit) self.os = patch('ansible.module_utils.basic.os').start() From 4bb4c7e68ed2bbac95b0d2cec3822b2ed8104b72 Mon Sep 17 00:00:00 2001 From: camradal Date: Fri, 13 May 2016 23:57:08 -0700 Subject: [PATCH 103/133] vCloud module utils error handling bug fixes (#15859) * Fix AttributeError that hides login errors * Typo fixes for vca error messages --- lib/ansible/module_utils/vca.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ansible/module_utils/vca.py b/lib/ansible/module_utils/vca.py index e895d28a7d..6c809887a9 100644 --- a/lib/ansible/module_utils/vca.py +++ b/lib/ansible/module_utils/vca.py @@ -111,7 +111,7 @@ class VcaAnsibleModule(AnsibleModule): def create_instance(self): service_type = self.params.get('service_type', DEFAULT_SERVICE_TYPE) - if service_type == 'vcd': + if service_type == 'vcd': host = self.params['host'] else: host = LOGIN_HOST[service_type] @@ -136,7 +136,7 @@ class VcaAnsibleModule(AnsibleModule): login_org = self.params['org'] if not self.vca.login(password=password, org=login_org): - self.fail('Login to VCA failed', response=self.vca.response.content) + self.fail('Login to VCA failed', response=self.vca.response) try: method_name = 'login_%s' % service_type @@ -145,7 +145,7 @@ class VcaAnsibleModule(AnsibleModule): except AttributeError: self.fail('no login method exists for service_type %s' % service_type) except VcaError, e: - self.fail(e.message, response=self.vca.response.content, **e.kwargs) + self.fail(e.message, response=self.vca.response, **e.kwargs) def login_vca(self): instance_id = self.params['instance_id'] @@ -160,14 +160,14 @@ class VcaAnsibleModule(AnsibleModule): org = self.params['org'] if not org: - raise VcaError('missing required or for service_type vchs') + raise VcaError('missing required org for service_type vchs') self.vca.login_to_org(service_id, org) def login_vcd(self): org = self.params['org'] if not org: - raise VcaError('missing required or for service_type vchs') + raise VcaError('missing required org for service_type vcd') if not self.vca.token: raise VcaError('unable to get token for service_type vcd') From 7f8a6d11c260f6b3c9892d78a3948372a27b0aad Mon Sep 17 00:00:00 2001 From: chouseknecht Date: Sat, 14 May 2016 09:26:32 -0400 Subject: [PATCH 104/133] Bumping core submodule ref. --- lib/ansible/modules/core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 298fd0ae56..a81d0a6809 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 298fd0ae56713e851fda06da13c5662f6be549dc +Subproject commit a81d0a6809e1053d2b70120843340e4f0cf2f110 From 56bb3ec6809ee28e626459db79ec6af4ce3b77d4 Mon Sep 17 00:00:00 2001 From: chouseknecht Date: Sat, 14 May 2016 09:37:27 -0400 Subject: [PATCH 105/133] Bump core submodule ref. --- lib/ansible/modules/core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index a81d0a6809..127d518011 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit a81d0a6809e1053d2b70120843340e4f0cf2f110 +Subproject commit 127d518011224289f7917fd3b2ec8ddf73c7dd17 From 9a69354b6a8b2191ebd9458fdb03eb7f5c232745 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Sat, 14 May 2016 07:34:38 -0700 Subject: [PATCH 106/133] Start a document for developing modules for python3 --- docsite/rst/developing_modules_python3.rst | 135 +++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 docsite/rst/developing_modules_python3.rst diff --git a/docsite/rst/developing_modules_python3.rst b/docsite/rst/developing_modules_python3.rst new file mode 100644 index 0000000000..c9d246d086 --- /dev/null +++ b/docsite/rst/developing_modules_python3.rst @@ -0,0 +1,135 @@ +=========================== +Porting Modules to Python 3 +=========================== + +Ansible modules are not the usual Python-3 porting exercise. There are two +factors that make it harder to port them than most code: + +1. Many modules need to run on Python-2.4 in addition to Python-3. +2. A lot of mocking has to go into unittesting a Python-3 module. So it's + harder to test that your porting has fixed everything or to make sure that + later commits haven't regressed. + +Which version of Python-3.x and which version of Python-2.x are our minimums? +============================================================================= + +The short answer is Python-3.4 and Python-2.4 but please read on for more +information. + +For Python-3 we are currently using Python-3.4 as a minimum. However, no long +term supported Linux distributions currently ship with Python-3. When that +occurs, we will probably take that as our minimum Python-3 version rather than +Python-3.4. Thus far, Python-3 has been adding small changes that make it +more compatible with Python-2 in its newer versions (For instance, Python-3.5 +added the ability to use percent-formatted byte strings.) so it should be more +pleasant to use a newer version of Python-3 if it's available. At some point +this will change but we'll just have to cross that bridge when we get to it. + +For Python-2 the default is for modules to run on Python-2.4. This allows +users with older distributions that are stuck on Python-2.4 to manage their +machines. Modules are allowed to drop support for Python-2.4 when one of +their dependent libraries require a higher version of python. This is not an +invitation to add unnecessary dependent libraries in order to force your +module to be usable only with a newer version of Python. Instead it is an +acknowledgment that some libraries (for instance, boto3 and docker-py) will +only function with newer Python. + +.. note:: When will we drop support for Python-2.4? + + The only long term supported distro that we know of with Python-2.4 is + RHEL5 (and its rebuilds like CentOS5) which is supported until April of + 2017. We will likely end our support for Python-2.4 in modules in an + Ansible release around that time. We know of no long term supported + distributions with Python-2.5 so the new minimum Python-2 version will + likely be Python-2.6. This will let us take advantage of the + forwards-compat features of Python-2.6 so porting and maintainance of + Python-2/Python-3 code will be easier after that. + +Supporting only Python-2 or only Python-3 +========================================= + +Sometimes a module's dependent libraries only run on Python-2 or only run on +Python-3. We do not yet have a strategy for these modules but we'll need to +come up with one. I see three possibilities: + +1. We treat these libraries like any other libraries that may not be installed + on the system. When we import them we check if the import was successful. + If so, then we continue. If not we return an error about the library being + missing. Users will have to find out that the library is unavailable on + their version of Python either by searching for the library on their own or + reading the requirements section in :command:`ansible-doc`. + +2. The shebang line is the only metadata that Ansible extracts from a module + so we may end up using that to specify what we mean. Something like + ``#!/usr/bin/python`` means the module will run on both Python-2 and + Python-3, ``#!/usr/bin/python2`` means the module will only run on + Python-2, and ``#!/usr/bin/python3`` means the module will only run on + Python-3. Ansible's code will need to be modified to accommodate this. + For :command:`python2`, if ``ansible_python2_interpreter`` is not set, it + will have to fallback to `` ansible_python_interpreter`` and if that's not + set, fallback to ``/usr/bin/python``. For :command:`python3`, Ansible + will have to first try ``ansible_python3_interpreter`` and then fallback to + ``/usr/bin/python3`` as normal. + +3. We add a way for Ansible to retrieve metadata about modules. The metadata + will include the version of Python that is required. + +Methods 2 and 3 will both require that we modify modules or otherwise add this +additional information somewhere. + +Tips, tricks, and idioms to adopt +================================= + +Exceptions +---------- + +In code which already needs Python-2.6+ (For instance, because a library it +depends on only runs on Python >= 2.6) it is okay to port directly to the new +exception catching syntax:: + + try: + a = 2/0 + except ValueError as e: + module.fail_json(msg="Tried to divide by zero!") + +For modules which also run on Python-2.4, we have to use an uglier +construction to make this work under both Python-2.4 and Python-3:: + + from ansible.module_utils.pycompat import get_exception + [...] + + try: + a = 2/0 + except ValueError: + e = get_exception() + module.fail_json(msg="Tried to divide by zero!") + +Octal numbers +------------- + +In Python-2.4, octal literals are specified as ``0755``. In Python-3, that is +invalid and octals must be specified as ``0o755``. To bridge this gap, +modules should create their octals like this:: + + # Can't use 0755 on Python-3 and can't use 0o755 on Python-2.4 + EXECUTABLE_PERMS = int('0755', 8) + + +Compile Test +------------ + +We have travis compiling all modules with various versions of Python to check +that the modules conform to the syntax at those versions. When you've +ported a module so that its syntax works with Python-3, we need to modify +.travis.yml so that the module is included in the syntax check. Here's the +relevant section of .travis.yml:: + + script: + [...] + - python3.4 -m compileall -fq system/ping.py + - python3.5 -m compileall -fq system/ping.py + +At the moment this is a whitelist. Just add your newly ported module to that +line. Eventually, not compiling on Python-3 will be the exception. When that +occurs, we will move to a blacklist for listing which modules do not compile +under Python-3. From 0cb05d8ac994ecb828cb9e4c88e92f3277e8ee1b Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Sat, 14 May 2016 07:51:13 -0700 Subject: [PATCH 107/133] Some Python-3 module_utils support --- lib/ansible/module_utils/basic.py | 18 +- lib/ansible/module_utils/pycompat.py | 44 ++ lib/ansible/module_utils/six.py | 577 +++++++++++++++++++++++++++ 3 files changed, 624 insertions(+), 15 deletions(-) create mode 100644 lib/ansible/module_utils/pycompat.py create mode 100644 lib/ansible/module_utils/six.py diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 14348adf40..54facf1e79 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -219,6 +219,9 @@ except ImportError: _literal_eval = literal_eval +# Backwards compat. There were present in basic.py before +from ansible.module_utils.pycompat import get_exception + # Internal global holding passed in params and constants. This is consulted # in case multiple AnsibleModules are created. Otherwise each AnsibleModule # would attempt to read from stdin. Other code should not use this directly @@ -253,21 +256,6 @@ EXEC_PERM_BITS = int('00111', 8) # execute permission bits DEFAULT_PERM = int('0666', 8) # default file permission bits -def get_exception(): - """Get the current exception. - - This code needs to work on Python 2.4 through 3.x, so we cannot use - "except Exception, e:" (SyntaxError on Python 3.x) nor - "except Exception as e:" (SyntaxError on Python 2.4-2.5). - Instead we must use :: - - except Exception: - e = get_exception() - - """ - return sys.exc_info()[1] - - def get_platform(): ''' what's the platform? example: Linux is a platform. ''' return platform.system() diff --git a/lib/ansible/module_utils/pycompat.py b/lib/ansible/module_utils/pycompat.py new file mode 100644 index 0000000000..6f51360fcc --- /dev/null +++ b/lib/ansible/module_utils/pycompat.py @@ -0,0 +1,44 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c) 2016, Toshio Kuratomi +# Copyright (c) 2015, Marius Gedminas +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +import sys + +def get_exception(): + """Get the current exception. + + This code needs to work on Python 2.4 through 3.x, so we cannot use + "except Exception, e:" (SyntaxError on Python 3.x) nor + "except Exception as e:" (SyntaxError on Python 2.4-2.5). + Instead we must use :: + + except Exception: + e = get_exception() + + """ + return sys.exc_info()[1] diff --git a/lib/ansible/module_utils/six.py b/lib/ansible/module_utils/six.py new file mode 100644 index 0000000000..85898ec712 --- /dev/null +++ b/lib/ansible/module_utils/six.py @@ -0,0 +1,577 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +# Copyright (c) 2010-2013 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.4.1" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) + # This is a bit ugly, but it avoids running this again. + delattr(tp, self.name) + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + + +class _MovedItems(types.ModuleType): + """Lazy loading of moved objects""" + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("winreg", "_winreg"), +] +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) +del attr + +moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves") + + + +class Module_six_moves_urllib_parse(types.ModuleType): + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +sys.modules[__name__ + ".moves.urllib_parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse") +sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib.parse") + + +class Module_six_moves_urllib_error(types.ModuleType): + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +sys.modules[__name__ + ".moves.urllib_error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib_error") +sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error") + + +class Module_six_moves_urllib_request(types.ModuleType): + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +sys.modules[__name__ + ".moves.urllib_request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib_request") +sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request") + + +class Module_six_moves_urllib_response(types.ModuleType): + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +sys.modules[__name__ + ".moves.urllib_response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib_response") +sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(types.ModuleType): + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +sys.modules[__name__ + ".moves.urllib_robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib_robotparser") +sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + parse = sys.modules[__name__ + ".moves.urllib_parse"] + error = sys.modules[__name__ + ".moves.urllib_error"] + request = sys.modules[__name__ + ".moves.urllib_request"] + response = sys.modules[__name__ + ".moves.urllib_response"] + robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"] + + +sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" + + _iterkeys = "keys" + _itervalues = "values" + _iteritems = "items" + _iterlists = "lists" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + _iterkeys = "iterkeys" + _itervalues = "itervalues" + _iteritems = "iteritems" + _iterlists = "iterlists" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +def iterkeys(d, **kw): + """Return an iterator over the keys of a dictionary.""" + return iter(getattr(d, _iterkeys)(**kw)) + +def itervalues(d, **kw): + """Return an iterator over the values of a dictionary.""" + return iter(getattr(d, _itervalues)(**kw)) + +def iteritems(d, **kw): + """Return an iterator over the (key, value) pairs of a dictionary.""" + return iter(getattr(d, _iteritems)(**kw)) + +def iterlists(d, **kw): + """Return an iterator over the (key, [values]) pairs of a dictionary.""" + return iter(getattr(d, _iterlists)(**kw)) + + +if PY3: + def b(s): + return s.encode("latin-1") + def u(s): + return s + unichr = chr + if sys.version_info[1] <= 1: + def int2byte(i): + return bytes((i,)) + else: + # This is about 2x faster than the implementation above on 3.2+ + int2byte = operator.methodcaller("to_bytes", 1, "big") + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO +else: + def b(s): + return s + def u(s): + return unicode(s, "unicode_escape") + unichr = unichr + int2byte = chr + def byte2int(bs): + return ord(bs[0]) + def indexbytes(buf, i): + return ord(buf[i]) + def iterbytes(buf): + return (ord(byte) for byte in buf) + import StringIO + StringIO = BytesIO = StringIO.StringIO +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +if PY3: + import builtins + exec_ = getattr(builtins, "exec") + + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + + print_ = getattr(builtins, "print") + del builtins + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + + def print_(*args, **kwargs): + """The new-style print function.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + def write(data): + if not isinstance(data, basestring): + data = str(data) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) + +_add_doc(reraise, """Reraise an exception.""") + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + return meta("NewBase", bases, {}) + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + for slots_var in orig_vars.get('__slots__', ()): + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper From b9aafb6f8972b12f564d2690d645a79da02a5b55 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Sat, 14 May 2016 07:56:06 -0700 Subject: [PATCH 108/133] Add six to developing_modules documentation --- docsite/rst/developing_modules_python3.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docsite/rst/developing_modules_python3.rst b/docsite/rst/developing_modules_python3.rst index c9d246d086..5d3c913512 100644 --- a/docsite/rst/developing_modules_python3.rst +++ b/docsite/rst/developing_modules_python3.rst @@ -114,6 +114,21 @@ modules should create their octals like this:: # Can't use 0755 on Python-3 and can't use 0o755 on Python-2.4 EXECUTABLE_PERMS = int('0755', 8) +Bundled six +----------- + +The third-party python-six library exists to help projects create code that +runs on both Python-2 and Python-3. Ansible includes version 1.4.1 in +module_utils so that other modules can use it without requiring that it is +installed on the remote system. To make use of it, import it like this:: + + from ansible.module_utils import six + +.. note:: Why version 1.4.1? + + six-1.4.1 is the last version of python-six to support Python-2.4. As + long as Ansible modules need to run on Python-2.4 we won't be able to + update the bundled copy of six. Compile Test ------------ From 95cf095222fef2ea6cde04142021fd7533eabadf Mon Sep 17 00:00:00 2001 From: feliksik Date: Sun, 15 May 2016 03:48:31 +0200 Subject: [PATCH 109/133] hashi_vault lookup: be more rebust, and allow fields with other name than 'value' (#13690) * more robust hashi_vault module, and allow querying specific field in secret-dict * allow fetching entire secret dict with trailing ':' * process comment by bcoca for PR #13690 --- lib/ansible/plugins/lookup/hashi_vault.py | 41 ++++++++++++++++++----- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/lib/ansible/plugins/lookup/hashi_vault.py b/lib/ansible/plugins/lookup/hashi_vault.py index 4e09d0c7b2..4c37d4e199 100644 --- a/lib/ansible/plugins/lookup/hashi_vault.py +++ b/lib/ansible/plugins/lookup/hashi_vault.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . # -# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}} +# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}} # # You can skip setting the url if you set the VAULT_ADDR environment variable # or if you want it to default to localhost:8200 @@ -47,9 +47,23 @@ class HashiVault: except ImportError: AnsibleError("Please pip install hvac to use this module") - self.url = kwargs.pop('url') - self.secret = kwargs.pop('secret') - self.token = kwargs.pop('token') + self.url = kwargs.get('url', ANSIBLE_HASHI_VAULT_ADDR) + + self.token = kwargs.get('token') + if self.token==None: + raise AnsibleError("No Vault Token specified") + + # split secret arg, which has format 'secret/hello:value' into secret='secret/hello' and secret_field='value' + s = kwargs.get('secret') + if s==None: + raise AnsibleError("No secret specified") + + s_f = s.split(':') + self.secret = s_f[0] + if len(s_f)>=2: + self.secret_field = s_f[1] + else: + self.secret_field = 'value' self.client = hvac.Client(url=self.url, token=self.token) @@ -62,20 +76,27 @@ class HashiVault: data = self.client.read(self.secret) if data is None: raise AnsibleError("The secret %s doesn't seem to exist" % self.secret) - else: - return data['data']['value'] + + if self.secret_field=='': # secret was specified with trailing ':' + return data['data'] + + if self.secret_field not in data['data']: + raise AnsibleError("The secret %s does not contain the field '%s'. " % (self.secret, self.secret_field)) + + return data['data'][self.secret_field] class LookupModule(LookupBase): - def run(self, terms, variables, **kwargs): - vault_args = terms[0].split(' ') vault_dict = {} ret = [] for param in vault_args: - key, value = param.split('=') + try: + key, value = param.split('=') + except ValueError as e: + raise AnsibleError("hashi_vault plugin needs key=value pairs, but received %s" % terms) vault_dict[key] = value vault_conn = HashiVault(**vault_dict) @@ -84,4 +105,6 @@ class LookupModule(LookupBase): key = term.split()[0] value = vault_conn.get() ret.append(value) + return ret + From e539b2003dd0070d6de00564b6aee4f42d1a88f0 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:02:38 +0200 Subject: [PATCH 110/133] Make the facts module run on netbsd (#15833) It currently fail with ansible/module_utils/facts.py\", line 357, in get_service_mgr_facts\r\nKeyError: 'distribution'\r\n" Since self.facts['distribution'] is used after, we need to make sure this is set by default and if needed, corrected somewhere for Linux. --- lib/ansible/module_utils/facts.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index 03121e6814..38d6cb9549 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -664,15 +664,11 @@ class Distribution(object): self.facts['distribution_release'] = platform.release() self.facts['distribution_version'] = platform.version() - systems_platform_working = ('NetBSD', 'FreeBSD') systems_implemented = ('AIX', 'HP-UX', 'Darwin', 'OpenBSD') - if self.system in systems_platform_working: - # the distribution is provided by platform module already and needs no fixes - pass + self.facts['distribution'] = self.system - elif self.system in systems_implemented: - self.facts['distribution'] = self.system + if self.system in systems_implemented: cleanedname = self.system.replace('-','') distfunc = getattr(self, 'get_distribution_'+cleanedname) distfunc() From 127a37f67c0e4d6bf4ee4da69d0499a2f99a0b37 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:04:43 +0200 Subject: [PATCH 111/133] Port the module snippet to python3 (#15870) vca depend on pyvcloud, who depend on PyYAML 3.10, which support python 2.5 as a minimum, cf https://github.com/vmware/pyvcloud/blob/master/requirements.txt and http://pyyaml.org/wiki/PyYAML vmware.py depend on PyVIM, who depend on python 2.6. So we can use the modern syntax for both of them. --- lib/ansible/module_utils/vca.py | 4 ++-- lib/ansible/module_utils/vmware.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ansible/module_utils/vca.py b/lib/ansible/module_utils/vca.py index 6c809887a9..070b2dec18 100644 --- a/lib/ansible/module_utils/vca.py +++ b/lib/ansible/module_utils/vca.py @@ -144,7 +144,7 @@ class VcaAnsibleModule(AnsibleModule): meth() except AttributeError: self.fail('no login method exists for service_type %s' % service_type) - except VcaError, e: + except VcaError as e: self.fail(e.message, response=self.vca.response, **e.kwargs) def login_vca(self): @@ -318,7 +318,7 @@ def vca_login(module): _vchs_login(vca, password, service, org) elif service_type == 'vcd': _vcd_login(vca, password, org) - except VcaError, e: + except VcaError as e: module.fail_json(msg=e.message, **e.kwargs) return vca diff --git a/lib/ansible/module_utils/vmware.py b/lib/ansible/module_utils/vmware.py index 24810e47a2..b2ac975039 100644 --- a/lib/ansible/module_utils/vmware.py +++ b/lib/ansible/module_utils/vmware.py @@ -194,9 +194,9 @@ def connect_to_api(module, disconnect_atexit=True): try: service_instance = connect.SmartConnect(host=hostname, user=username, pwd=password) - except vim.fault.InvalidLogin, invalid_login: + except vim.fault.InvalidLogin as invalid_login: module.fail_json(msg=invalid_login.msg, apierror=str(invalid_login)) - except requests.ConnectionError, connection_error: + except requests.ConnectionError as connection_error: if '[SSL: CERTIFICATE_VERIFY_FAILED]' in str(connection_error) and not validate_certs: context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_NONE From 97f16b77007598f130280f68cded618b1cdf677b Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:10:07 +0200 Subject: [PATCH 112/133] Port shell snippet to python3/2.4 compatible syntax (#15874) --- lib/ansible/module_utils/shell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/shell.py b/lib/ansible/module_utils/shell.py index a01b41b0c5..9e244cd638 100644 --- a/lib/ansible/module_utils/shell.py +++ b/lib/ansible/module_utils/shell.py @@ -31,6 +31,7 @@ try: except ImportError: HAS_PARAMIKO = False +from ansible.module_utils.basic import get_exception ANSI_RE = re.compile(r'(\x1b\[\?1h\x1b=)') @@ -135,7 +136,8 @@ class Shell(object): if self.read(window): resp = self.strip(recv.getvalue()) return self.sanitize(cmd, resp) - except ShellError, exc: + except ShellError: + exc = get_exception() exc.command = cmd raise From a4f6fc0dc200c49c7b3a8c047a7cbc322495e785 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:10:35 +0200 Subject: [PATCH 113/133] Port docker_common.py to py3 compatible syntax (#15877) Since docker-py depend on python 2.6 (cf their tox.ini), we do not need to make it python 2.4 compatible. --- lib/ansible/module_utils/docker_common.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/ansible/module_utils/docker_common.py b/lib/ansible/module_utils/docker_common.py index c324883863..508bf12347 100644 --- a/lib/ansible/module_utils/docker_common.py +++ b/lib/ansible/module_utils/docker_common.py @@ -37,7 +37,7 @@ try: from docker.constants import DEFAULT_TIMEOUT_SECONDS, DEFAULT_DOCKER_API_VERSION from docker.utils.types import Ulimit, LogConfig from docker import auth -except ImportError, exc: +except ImportError as exc: HAS_DOCKER_ERROR = str(exc) HAS_DOCKER_PY = False @@ -161,9 +161,9 @@ class AnsibleDockerClient(Client): try: super(AnsibleDockerClient, self).__init__(**self._connect_params) - except APIError, exc: + except APIError as exc: self.fail("Docker API error: %s" % exc) - except Exception, exc: + except Exception as exc: self.fail("Error connecting: %s" % exc) def log(self, msg, pretty_print=False): @@ -262,7 +262,7 @@ class AnsibleDockerClient(Client): try: tls_config = TLSConfig(**kwargs) return tls_config - except TLSParameterError, exc: + except TLSParameterError as exc: self.fail("TLS config error: %s" % exc) def _get_connect_params(self): @@ -372,9 +372,9 @@ class AnsibleDockerClient(Client): if container['Id'] == name: result = container break - except SSLError, exc: + except SSLError as exc: self._handle_ssl_error(exc) - except Exception, exc: + except Exception as exc: self.fail("Error retrieving container list: %s" % exc) if result is not None: @@ -382,7 +382,7 @@ class AnsibleDockerClient(Client): self.log("Inspecting container Id %s" % result['Id']) result = self.inspect_container(container=result['Id']) self.log("Completed container inspection") - except Exception, exc: + except Exception as exc: self.fail("Error inspecting container: %s" % exc) return result @@ -411,7 +411,7 @@ class AnsibleDockerClient(Client): if len(images) == 1: try: inspection = self.inspect_image(images[0]['Id']) - except Exception, exc: + except Exception as exc: self.fail("Error inspecting image %s:%s - %s" % (name, tag, str(exc))) return inspection @@ -455,7 +455,7 @@ class AnsibleDockerClient(Client): error_detail.get('message'))) else: self.fail("Error pulling %s - %s" % (name, line.get('error'))) - except Exception, exc: + except Exception as exc: self.fail("Error pulling image %s:%s - %s" % (name, tag, str(exc))) return self.find_image(name, tag) From fae492324e4f4667d44be31893baa795ad7689d6 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:11:15 +0200 Subject: [PATCH 114/133] Port the rest of the file to the 2.4/3 compatible syntax (#15873) Since the modules can use a paramiko transport (ergo python 2.4 syntax), we need to keep compat with 2.4 and python 3, so we need to use the get_exception trick, even if the various juniper libraries are not compatible with 2.4. --- lib/ansible/module_utils/junos.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/ansible/module_utils/junos.py b/lib/ansible/module_utils/junos.py index bf1a2f955d..5654816902 100644 --- a/lib/ansible/module_utils/junos.py +++ b/lib/ansible/module_utils/junos.py @@ -158,7 +158,8 @@ class Netconf(object): self.config = Config(self.device) - except Exception, exc: + except Exception: + exc = get_exception() self._fail('unable to connect to %s: %s' % (host, str(exc))) def run_commands(self, commands, **kwargs): @@ -169,9 +170,11 @@ class Netconf(object): try: resp = self.device.cli(command=cmd, format=fmt) response.append(resp) - except (ValueError, RpcError), exc: + except (ValueError, RpcError): + exc = get_exception() self._fail('Unable to get cli output: %s' % str(exc)) - except Exception, exc: + except Exception: + exc = get_exception() self._fail('Uncaught exception - please report: %s' % str(exc)) return response @@ -180,14 +183,16 @@ class Netconf(object): try: self.config.unlock() self._locked = False - except UnlockError, exc: + except UnlockError: + exc = get_exception() self.module.log('unable to unlock config: {0}'.format(str(exc))) def lock_config(self): try: self.config.lock() self._locked = True - except LockError, exc: + except LockError: + exc = get_exception() self.module.log('unable to lock config: {0}'.format(str(exc))) def check_config(self): @@ -200,7 +205,8 @@ class Netconf(object): if confirm and confirm > 0: kwargs['confirm'] = confirm return self.config.commit(**kwargs) - except CommitError, exc: + except CommitError: + exc = get_exception() msg = 'Unable to commit configuration: {0}'.format(str(exc)) self._fail(msg=msg) @@ -215,7 +221,8 @@ class Netconf(object): try: self.config.load(candidate, format=format, merge=merge, overwrite=overwrite) - except ConfigLoadError, exc: + except ConfigLoadError: + exc = get_exception() msg = 'Unable to load config: {0}'.format(str(exc)) self._fail(msg=msg) @@ -234,7 +241,8 @@ class Netconf(object): try: result = self.config.rollback(identifier) - except Exception, exc: + except Exception: + exc = get_exception() msg = 'Unable to rollback config: {0}'.format(str(exc)) self._fail(msg=msg) From cdef2f5db69e7e9eabaa4708d0783df4fa92b60e Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:11:31 +0200 Subject: [PATCH 115/133] Port the gce snippet to a python 2.6 to 3 compatible syntax (#15872) Since it depend on libcloud and libcloud requirements include python 2.6 since libcloud 0.4.0 (https://libcloud.apache.org/about.html), which was released in 2011 Q2, and GCE drivers were added in 2013, we can't run a libcloud version with GCE support on 2.4. --- lib/ansible/module_utils/gce.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ansible/module_utils/gce.py b/lib/ansible/module_utils/gce.py index a57fdb46d5..0008681f1e 100644 --- a/lib/ansible/module_utils/gce.py +++ b/lib/ansible/module_utils/gce.py @@ -100,7 +100,7 @@ def gce_connect(module, provider=None): module.fail_json(msg='Using JSON credentials but libcloud minimum version not met. ' 'Upgrade to libcloud>=0.17.0.') return None - except ValueError, e: + except ValueError as e: # Not JSON pass @@ -114,9 +114,9 @@ def gce_connect(module, provider=None): project=project_id) gce.connection.user_agent_append("%s/%s" % ( USER_AGENT_PRODUCT, USER_AGENT_VERSION)) - except (RuntimeError, ValueError), e: + except (RuntimeError, ValueError) as e: module.fail_json(msg=str(e), changed=False) - except Exception, e: + except Exception as e: module.fail_json(msg=unexpected_error_msg(e), changed=False) return gce From 1e60fb6c1982d35d6d81a4b78611c0bb00351b70 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:33:00 +0200 Subject: [PATCH 116/133] Add my irc nick (#15881) --- docsite/rst/committer_guidelines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/committer_guidelines.rst b/docsite/rst/committer_guidelines.rst index 6ca974eeba..92cd8c45f5 100644 --- a/docsite/rst/committer_guidelines.rst +++ b/docsite/rst/committer_guidelines.rst @@ -89,7 +89,7 @@ Individuals who've been asked to become a part of this group have generally been +---------------------+----------------------+--------------------+----------------------+ | Abhijit Menon-Sen | amenonsen | crab | | +---------------------+----------------------+--------------------+----------------------+ -| Michael Scherer | mscherer | | | +| Michael Scherer | mscherer | misc | | +---------------------+----------------------+--------------------+----------------------+ | RenĂ© Moser | resmo | resmo | | +---------------------+----------------------+--------------------+----------------------+ From eb52dc9af0e4d83484e19232dbf4c8dbdaaa7911 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:42:28 +0200 Subject: [PATCH 117/133] Port azure_rm_common.py to py3 syntax (#15880) Since the rest of the file already use a non 2.4 syntax (such as format), I didn't bother using the 2.4 syntax for exceptions. --- lib/ansible/module_utils/azure_rm_common.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/ansible/module_utils/azure_rm_common.py b/lib/ansible/module_utils/azure_rm_common.py index 526655fcb7..6fe285e26a 100644 --- a/lib/ansible/module_utils/azure_rm_common.py +++ b/lib/ansible/module_utils/azure_rm_common.py @@ -88,7 +88,7 @@ try: from azure.mgmt.compute.compute_management_client import ComputeManagementClient,\ ComputeManagementClientConfiguration from azure.storage.cloudstorageaccount import CloudStorageAccount -except ImportError, exc: +except ImportError as exc: HAS_AZURE_EXC = exc HAS_AZURE = False @@ -323,7 +323,7 @@ class AzureRMModuleBase(object): return self.rm_client.resource_groups.get(resource_group) except CloudError: self.fail("Parameter error: resource group {0} not found".format(resource_group)) - except Exception, exc: + except Exception as exc: self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc))) def _get_profile(self, profile="default"): @@ -331,7 +331,7 @@ class AzureRMModuleBase(object): try: config = ConfigParser.ConfigParser() config.read(path) - except Exception, exc: + except Exception as exc: self.fail("Failed to access {0}. Check that the file exists and you have read " "access. {1}".format(path, str(exc))) credentials = dict() @@ -418,7 +418,7 @@ class AzureRMModuleBase(object): self.log("Waiting for {0} sec".format(delay)) poller.wait(timeout=delay) return poller.result() - except Exception, exc: + except Exception as exc: self.log(str(exc)) raise @@ -465,13 +465,13 @@ class AzureRMModuleBase(object): account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name, storage_account_name) keys['key1'] = account_keys.key1 keys['key2'] = account_keys.key2 - except Exception, exc: + except Exception as exc: self.fail("Error getting keys for account {0} - {1}".format(storage_account_name, str(exc))) try: self.log('Create blob service') return CloudStorageAccount(storage_account_name, keys['key1']).create_block_blob_service() - except Exception, exc: + except Exception as exc: self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name, str(exc))) @@ -508,7 +508,7 @@ class AzureRMModuleBase(object): self.log('Creating default public IP {0}'.format(public_ip_name)) try: poller = self.network_client.public_ip_addresses.create_or_update(resource_group, public_ip_name, params) - except Exception, exc: + except Exception as exc: self.fail("Error creating {0} - {1}".format(public_ip_name, str(exc))) return self.get_poller_result(poller) @@ -578,7 +578,7 @@ class AzureRMModuleBase(object): poller = self.network_client.network_security_groups.create_or_update(resource_group, security_group_name, parameters) - except Exception, exc: + except Exception as exc: self.fail("Error creating default security rule {0} - {1}".format(security_group_name, str(exc))) return self.get_poller_result(poller) @@ -589,7 +589,7 @@ class AzureRMModuleBase(object): # time we attempt to use the requested client. resource_client = self.rm_client resource_client.providers.register(key) - except Exception, exc: + except Exception as exc: self.fail("One-time registration of {0} failed - {1}".format(key, str(exc))) @property From 10edaabed54467035549e514b5cbe4d78bf38552 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 14:44:51 +0200 Subject: [PATCH 118/133] Port module_utils/ec2.py to python 3 syntax (#15879) Since boto is considered as python 2.6 only (cf https://github.com/ansible/ansible/blob/devel/test/utils/run_tests.sh#L15 ), no need to use the 2.4 compatible syntax. --- lib/ansible/module_utils/ec2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index bd4479fc59..b0cf050c08 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -226,13 +226,13 @@ def ec2_connect(module): if region: try: ec2 = connect_to_aws(boto.ec2, region, **boto_params) - except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e: + except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e: module.fail_json(msg=str(e)) # Otherwise, no region so we fallback to the old connection method elif ec2_url: try: ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params) - except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e: + except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e: module.fail_json(msg=str(e)) else: module.fail_json(msg="Either region or ec2_url must be specified") From 11b3a1b8ce12d907f5a7a931a3827a9250aa7481 Mon Sep 17 00:00:00 2001 From: toromoti Date: Mon, 16 May 2016 22:48:54 +0900 Subject: [PATCH 119/133] change to a more easy-to-understand example (#15863) --- docsite/rst/playbooks_loops.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docsite/rst/playbooks_loops.rst b/docsite/rst/playbooks_loops.rst index a56af0b713..eaad679c51 100644 --- a/docsite/rst/playbooks_loops.rst +++ b/docsite/rst/playbooks_loops.rst @@ -556,7 +556,7 @@ Ansible by default sets the loop variable `item` for each loop, which causes the As of Ansible 2.1, the `loop_control` option can be used to specify the name of the variable to be used for the loop:: # main.yml - - include: test.yml outer_loop="{{ outer_item }}" + - include: inner.yml with_items: - 1 - 2 @@ -565,7 +565,7 @@ As of Ansible 2.1, the `loop_control` option can be used to specify the name of loop_var: outer_item # inner.yml - - debug: msg="outer item={{ outer_loop }} inner item={{ item }}" + - debug: msg="outer item={{ outer_item }} inner item={{ item }}" with_items: - a - b @@ -583,7 +583,7 @@ Because `loop_control` is not available in Ansible 2.0, when using an include wi for `item`:: # main.yml - - include: test.yml + - include: inner.yml with_items: - 1 - 2 From 8de25db6813eaa5c46b86e709b3925035e6f9995 Mon Sep 17 00:00:00 2001 From: Kei Nohguchi Date: Mon, 16 May 2016 07:05:08 -0700 Subject: [PATCH 120/133] net_template.py: Fix jinja2 template file search path (#15134) The change is needed to support the multiple include statements inside the jinja2 template file, as in '{% include ['another.j2'] %}'. statement. I need this capability, as OpenSwitch `switch` role needs to handle multiple *.j2 files and supporting the include statement inside jinja2 file is essential, otherwise I need to combine multiple template files into a single file, which easily causes conflicts between developers working on different parts of the teamplate, ports and interface. --- lib/ansible/plugins/action/net_template.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/ansible/plugins/action/net_template.py b/lib/ansible/plugins/action/net_template.py index c626c8dc6b..3d27cac5f2 100644 --- a/lib/ansible/plugins/action/net_template.py +++ b/lib/ansible/plugins/action/net_template.py @@ -93,6 +93,17 @@ class ActionModule(ActionBase): except IOError: return dict(failed=True, msg='unable to load src file') + # Create a template search path in the following order: + # [working_path, self_role_path, dependent_role_paths, dirname(source)] + searchpath = [working_path] + if self._task._role is not None: + searchpath.append(self._task._role._role_path) + dep_chain = self._task._block.get_dep_chain() + if dep_chain is not None: + for role in dep_chain: + searchpath.append(role._role_path) + searchpath.append(os.path.dirname(source)) + self._templar.environment.loader.searchpath = searchpath self._task.args['src'] = self._templar.template(template_data) From 8680cc7156c41191347b57a776316ffe9f1eccbb Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 16:13:49 +0200 Subject: [PATCH 121/133] Fix pkgin detection on NetBSD 6 and 7 (#15834) Since this is now the default package manager, it got moved to another location on Netbsd : netbsd# type pkgin pkgin is a tracked alias for /usr/pkg/bin/pkgin netbsd# uname -a NetBSD netbsd.example.org 6.1.4 NetBSD 6.1.4 (GENERIC) amd64 But since the package manager is also used outside of NetBSD, we have to keep the /opt/local path too. --- lib/ansible/module_utils/facts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index 38d6cb9549..8113f5ecba 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -156,6 +156,7 @@ class Facts(object): { 'path' : '/usr/sbin/urpmi', 'name' : 'urpmi' }, { 'path' : '/usr/bin/pacman', 'name' : 'pacman' }, { 'path' : '/bin/opkg', 'name' : 'opkg' }, + { 'path' : '/usr/pkg/bin/pkgin', 'name' : 'pkgin' }, { 'path' : '/opt/local/bin/pkgin', 'name' : 'pkgin' }, { 'path' : '/opt/local/bin/port', 'name' : 'macports' }, { 'path' : '/usr/local/bin/brew', 'name' : 'homebrew' }, From aaaab4245667ae2401ffdb028d500f7740c3439c Mon Sep 17 00:00:00 2001 From: Chris Mague Date: Mon, 16 May 2016 07:44:35 -0700 Subject: [PATCH 122/133] update documentation (#15838) --- contrib/inventory/consul_io.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/contrib/inventory/consul_io.py b/contrib/inventory/consul_io.py index 5af7188a52..83a1cf5828 100755 --- a/contrib/inventory/consul_io.py +++ b/contrib/inventory/consul_io.py @@ -84,9 +84,9 @@ to retrieve the kv_groups and kv_metadata based on your consul configuration. This is used to lookup groups for a node in the key value store. It specifies a path to which each discovered node's name will be added to create a key to query the key/value store. There it expects to find a comma separated list of group -names to which the node should be added e.g. if the inventory contains -'nyc-web-1' and kv_groups = 'ansible/groups' then the key -'v1/kv/ansible/groups/nyc-web-1' will be queried for a group list. If this query +names to which the node should be added e.g. if the inventory contains node +'nyc-web-1' in datacenter 'nyc-dc1' and kv_groups = 'ansible/groups' then the key +'ansible/groups/nyc-dc1/nyc-web-1' will be queried for a group list. If this query returned 'test,honeypot' then the node address to both groups. 'kv_metadata': @@ -94,7 +94,9 @@ names to which the node should be added e.g. if the inventory contains kv_metadata is used to lookup metadata for each discovered node. Like kv_groups above it is used to build a path to lookup in the kv store where it expects to find a json dictionary of metadata entries. If found, each key/value pair in the -dictionary is added to the metadata for the node. +dictionary is added to the metadata for the node. eg node 'nyc-web-1' in datacenter +'nyc-dc1' and kv_metadata = 'ansible/metadata', then the key +'ansible/groups/nyc-dc1/nyc-web-1' should contain '{"databse": "postgres"}' 'availability': From cc61531a7445dc1d93f772a08f737613c644b087 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Mon, 16 May 2016 16:51:48 +0200 Subject: [PATCH 123/133] Do not test vca and vmware.py for py2.4 (#15887) Since both of them depend on libraries not working on python 2.4, we shouldn't restrict ourself on 2.4, cf https://github.com/ansible/ansible/pull/15870 --- test/utils/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh index e2bdef321e..a63cf24ddc 100755 --- a/test/utils/run_tests.sh +++ b/test/utils/run_tests.sh @@ -12,7 +12,7 @@ if [ "${TARGET}" = "sanity" ]; then ./test/code-smell/boilerplate.sh ./test/code-smell/required-and-default-attributes.sh if test x"$TOXENV" != x'py24' ; then tox ; fi - if test x"$TOXENV" = x'py24' ; then python2.4 -V && python2.4 -m compileall -fq -x 'module_utils/(a10|rax|openstack|ec2|gce|docker_common|azure_rm_common).py' lib/ansible/module_utils ; fi + if test x"$TOXENV" = x'py24' ; then python2.4 -V && python2.4 -m compileall -fq -x 'module_utils/(a10|rax|openstack|ec2|gce|docker_common|azure_rm_common|vca|vmware).py' lib/ansible/module_utils ; fi else if [ ! -e /tmp/cid_httptester ]; then docker pull sivel/httptester From ba63ccb880ca2b683a6bf879cb8397118c065a72 Mon Sep 17 00:00:00 2001 From: Rob Date: Tue, 17 May 2016 02:19:41 +1000 Subject: [PATCH 124/133] Handle case of both Key and key (#15468) --- lib/ansible/module_utils/ec2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index b0cf050c08..3b2714eb53 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -364,7 +364,10 @@ def boto3_tag_list_to_ansible_dict(tags_list): tags_dict = {} for tag in tags_list: - tags_dict[tag['Key']] = tag['Value'] + if 'key' in tag: + tags_dict[tag['key']] = tag['value'] + elif 'Key' in tag: + tags_dict[tag['Key']] = tag['Value'] return tags_dict From c648c95eb74fceb4478d674242603f6db037796d Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 16 May 2016 14:38:57 -0400 Subject: [PATCH 125/133] made format more flexible and allow for non dict entries --- lib/ansible/inventory/yaml.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/ansible/inventory/yaml.py b/lib/ansible/inventory/yaml.py index 620ace0d88..552c796071 100644 --- a/lib/ansible/inventory/yaml.py +++ b/lib/ansible/inventory/yaml.py @@ -27,6 +27,7 @@ from ansible.inventory.group import Group from ansible.inventory.expand_hosts import detect_range from ansible.inventory.expand_hosts import expand_hostname_range from ansible.parsing.utils.addresses import parse_address +from ansible.compat.six import string_types class InventoryParser(object): """ @@ -77,6 +78,11 @@ class InventoryParser(object): self.groups[group] = Group(name=group) if isinstance(group_data, dict): + #make sure they are dicts + for section in ['vars', 'children', 'hosts']: + if section in group_data and isinstance(group_data[section], string_types): + group_data[section] = { group_data[section]: None} + if 'vars' in group_data: for var in group_data['vars']: if var != 'ansible_group_priority': From 83d0a15588d1a2070217711c240caf027b6c4a83 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Mon, 16 May 2016 14:54:26 -0400 Subject: [PATCH 126/133] bugfix for issue with trying to eval contains with non-string fixes ansible/ansible-modules-core#3502 --- lib/ansible/module_utils/netcfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/netcfg.py b/lib/ansible/module_utils/netcfg.py index 68607f525c..20a3d3e681 100644 --- a/lib/ansible/module_utils/netcfg.py +++ b/lib/ansible/module_utils/netcfg.py @@ -382,7 +382,7 @@ class Conditional(object): return self.number(value) <= self.value def contains(self, value): - return self.value in value + return str(self.value) in value From 2cd17b66be15f0e089164d5a8abec3b6a49db501 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 16 May 2016 14:50:28 -0500 Subject: [PATCH 127/133] Updates links to OpenStack docs to specific pages (#15857) While a redirect is in place for both the links to the docs.openstack.org site, a more precise link is helpful. --- docsite/rst/intro_dynamic_inventory.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index d851bd190a..007540875d 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -253,13 +253,13 @@ Source an OpenStack RC file:: .. note:: - An OpenStack RC file contains the environment variables required by the client tools to establish a connection with the cloud provider, such as the authentication URL, user name, password and region name. For more information on how to download, create or source an OpenStack RC file, please refer to http://docs.openstack.org/cli-reference/content/cli_openrc.html. + An OpenStack RC file contains the environment variables required by the client tools to establish a connection with the cloud provider, such as the authentication URL, user name, password and region name. For more information on how to download, create or source an OpenStack RC file, please refer to `Set environment variables using the OpenStack RC file `_. You can confirm the file has been successfully sourced by running a simple command, such as `nova list` and ensuring it return no errors. .. note:: - The OpenStack command line clients are required to run the `nova list` command. For more information on how to install them, please refer to http://docs.openstack.org/cli-reference/content/install_clients.html. + The OpenStack command line clients are required to run the `nova list` command. For more information on how to install them, please refer to `Install the OpenStack command-line clients `_. You can test the OpenStack dynamic inventory script manually to confirm it is working as expected:: From 376fc21f92e9faf01d582a3e4c4948e5f198cc1d Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Tue, 17 May 2016 02:00:21 +0200 Subject: [PATCH 128/133] Add a exception for module_utils/six.py regarding code smell (#15878) Since six replace urlopen, it is normal to trigger the test, hence the exception --- test/code-smell/replace-urlopen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/code-smell/replace-urlopen.sh b/test/code-smell/replace-urlopen.sh index 725bcab4d3..3973ca64b1 100755 --- a/test/code-smell/replace-urlopen.sh +++ b/test/code-smell/replace-urlopen.sh @@ -3,7 +3,7 @@ BASEDIR=${1-"."} URLLIB_USERS=$(find "$BASEDIR" -name '*.py' -exec grep -H urlopen \{\} \;) -URLLIB_USERS=$(echo "$URLLIB_USERS" | sed '/\(\n\|lib\/ansible\/module_utils\/urls.py\|lib\/ansible\/compat\/six\/_six.py\|.tox\)/d') +URLLIB_USERS=$(echo "$URLLIB_USERS" | sed '/\(\n\|lib\/ansible\/module_utils\/urls.py\|lib\/ansible\/module_utils\/six.py\|lib\/ansible\/compat\/six\/_six.py\|.tox\)/d') URLLIB_USERS=$(echo "$URLLIB_USERS" | sed '/^[^:]\+:#/d') if test -n "$URLLIB_USERS" ; then printf "$URLLIB_USERS" From fb2355e47d6567b497b083522e8228c1cb4b5be8 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Tue, 17 May 2016 02:30:25 +0200 Subject: [PATCH 129/133] Port rax.py to python3 compatible syntax (#15875) Since the pyrax website say that only python 2.7 is tested, I do not think it is worth to aim for python 2.4 compatibility for the various rackspace modules. --- lib/ansible/module_utils/rax.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ansible/module_utils/rax.py b/lib/ansible/module_utils/rax.py index ad7afeb3ec..213a215556 100644 --- a/lib/ansible/module_utils/rax.py +++ b/lib/ansible/module_utils/rax.py @@ -163,7 +163,7 @@ def rax_find_volume(module, rax_module, name): volume = cbs.find(name=name) except rax_module.exc.NotFound: volume = None - except Exception, e: + except Exception as e: module.fail_json(msg='%s' % e) return volume @@ -302,7 +302,7 @@ def setup_rax_module(module, rax_module, region_required=True): os.environ.get('RAX_CREDS_FILE')) region = (region or os.environ.get('RAX_REGION') or rax_module.get_setting('region')) - except KeyError, e: + except KeyError as e: module.fail_json(msg='Unable to load %s' % e.message) try: @@ -317,7 +317,7 @@ def setup_rax_module(module, rax_module, region_required=True): rax_module.set_credential_file(credentials, region=region) else: raise Exception('No credentials supplied!') - except Exception, e: + except Exception as e: if e.message: msg = str(e.message) else: From be928ef71ae018ecfe7417d256a96e45c68a2e91 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 17 May 2016 08:16:10 +0100 Subject: [PATCH 130/133] add krb5-user to list of packages to install for ubuntu following this post on the google group: https://groups.google.com/forum/#!topic/ansible-project/dLhLtnuyzak --- docsite/rst/intro_windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/intro_windows.rst b/docsite/rst/intro_windows.rst index 77e5a12618..316ac6608b 100644 --- a/docsite/rst/intro_windows.rst +++ b/docsite/rst/intro_windows.rst @@ -44,7 +44,7 @@ Installing python-kerberos dependencies yum -y install python-devel krb5-devel krb5-libs krb5-workstation # Via Apt (Ubuntu) - sudo apt-get install python-dev libkrb5-dev + sudo apt-get install python-dev libkrb5-dev krb5-user # Via Portage (Gentoo) emerge -av app-crypt/mit-krb5 From 42d8ababcbb6c6368bea48ab8a61cb32849eac52 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Tue, 17 May 2016 19:50:28 +0200 Subject: [PATCH 131/133] Start to test module_utils for python 3 syntax (#15882) --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 60b446d955..267a5a5a7e 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,8 @@ commands = python --version py26: python -m compileall -fq -x 'test/samples|contrib/inventory/vagrant.py' lib test contrib py27: python -m compileall -fq -x 'test/samples' lib test contrib - py34: python -m compileall -fq -x 'test/samples|lib/ansible/module_utils|lib/ansible/modules' lib test contrib - py35: python -m compileall -fq -x 'test/samples|lib/ansible/module_utils|lib/ansible/modules' lib test contrib + py34: python -m compileall -fq -x 'test/samples|lib/ansible/modules' lib test contrib + py35: python -m compileall -fq -x 'test/samples|lib/ansible/modules' lib test contrib make tests From 73a2bddf8453848d3b02a56b5ea637cd9ec392ea Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 17 May 2016 10:51:14 -0700 Subject: [PATCH 132/133] Update submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 127d518011..92bf802cb8 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 127d518011224289f7917fd3b2ec8ddf73c7dd17 +Subproject commit 92bf802cb82844783a2b678b0e709bdd82c1103d diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index f953d5dc0c..e710dc47fe 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit f953d5dc0c1418dcdc5db72e8fb630b6aa83997f +Subproject commit e710dc47fe35fa2e05f57c184f34e2763f9ac864 From 42f6114b6162e976529a0fac5548e66ee8eb0653 Mon Sep 17 00:00:00 2001 From: nitzmahone Date: Tue, 17 May 2016 13:40:40 -0700 Subject: [PATCH 133/133] fix windows integration tests to run under kerberos users --- test/integration/roles/test_win_get_url/defaults/main.yml | 3 +-- test/integration/roles/test_win_get_url/tasks/main.yml | 8 ++++++++ test/integration/roles/test_win_script/defaults/main.yml | 1 - test/integration/roles/test_win_script/tasks/main.yml | 8 ++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/test/integration/roles/test_win_get_url/defaults/main.yml b/test/integration/roles/test_win_get_url/defaults/main.yml index 6e507ecf31..c7a90e599f 100644 --- a/test/integration/roles/test_win_get_url/defaults/main.yml +++ b/test/integration/roles/test_win_get_url/defaults/main.yml @@ -1,7 +1,6 @@ --- test_win_get_url_link: http://docs.ansible.com -test_win_get_url_path: "C:\\Users\\{{ansible_ssh_user}}\\docs_index.html" test_win_get_url_invalid_link: http://docs.ansible.com/skynet_module.html test_win_get_url_invalid_path: "Q:\\Filez\\Cyberdyne.html" -test_win_get_url_dir_path: "C:\\Users\\{{ansible_ssh_user}}" +test_win_get_url_path: "{{ test_win_get_url_dir_path }}\\docs_index.html" \ No newline at end of file diff --git a/test/integration/roles/test_win_get_url/tasks/main.yml b/test/integration/roles/test_win_get_url/tasks/main.yml index b0705eabd5..52e49672d2 100644 --- a/test/integration/roles/test_win_get_url/tasks/main.yml +++ b/test/integration/roles/test_win_get_url/tasks/main.yml @@ -16,6 +16,14 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . +- name: get tempdir path + raw: $env:TEMP + register: tempdir + +- name: set output path dynamically + set_fact: + test_win_get_url_dir_path: "{{ tempdir.stdout_lines[0] }}" + - name: remove test file if it exists raw: > PowerShell -Command Remove-Item "{{test_win_get_url_path}}" -Force diff --git a/test/integration/roles/test_win_script/defaults/main.yml b/test/integration/roles/test_win_script/defaults/main.yml index 90b756af0a..a2c6475e75 100644 --- a/test/integration/roles/test_win_script/defaults/main.yml +++ b/test/integration/roles/test_win_script/defaults/main.yml @@ -3,4 +3,3 @@ # Parameters to pass to test scripts. test_win_script_value: VaLuE test_win_script_splat: "@{This='THIS'; That='THAT'; Other='OTHER'}" -test_win_script_filename: "C:/Users/{{ansible_ssh_user}}/testing_win_script.txt" diff --git a/test/integration/roles/test_win_script/tasks/main.yml b/test/integration/roles/test_win_script/tasks/main.yml index 46f91f13f8..6cfa84ec00 100644 --- a/test/integration/roles/test_win_script/tasks/main.yml +++ b/test/integration/roles/test_win_script/tasks/main.yml @@ -16,6 +16,14 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . +- name: get tempdir path + raw: $env:TEMP + register: tempdir + +- name: set script path dynamically + set_fact: + test_win_script_filename: "{{ tempdir.stdout_lines[0] }}/testing_win_script.txt" + - name: run simple test script script: test_script.ps1 register: test_script_result