async: use async_dir for the async results file directory (#45461)

* win async: use async_dir for the async results file directory

* tried to unify POSIX and PowerShell async implementations of async_dir

* fix sanity issue

changelogs/fragments/async-dir.yaml

@@ -0,0 +1,7 @@
+minor_changes:
+- windows async - change default directory from ``$env:TEMP\.ansible_async`` to ``$env:USERPROFILE\.ansible_async`` to match the POSIX standard.
+- windows async - async directory is now controlled by the ``async_dir`` shell option and not ``remote_tmp`` to match the POSIX standard.
+bugfixes:
+- async - fixed issue where the shell option ``async_dir`` was not being used when setting the async directory.
+deprecated_features:
+- async - setting the async directory using ``ANSIBLE_ASYNC_DIR`` as an environment key in a task or play is deprecated and will be removed in Ansible 2.12. Set a var name ``ansible_async_dir`` instead.

docs/docsite/rst/porting_guides/porting_guide_2.8.rst

@@ -43,7 +43,15 @@ By default in Ansible 2.7, or with ``AGNOSTIC_BECOME_PROMPT=False`` in Ansible 2
 Deprecated
 ==========
 
-No notable changes.
+* Setting the async directory using ``ANSIBLE_ASYNC_DIR`` as an task/play environment key is deprecated and will be
+  removed in Ansible 2.12. You can achieve the same result by setting ``ansible_async_dir`` as a variable like::
+
+      - name: run task with custom async directory
+        command: sleep 5
+        async: 10
+        vars:
+          ansible_aync_dir: /tmp/.ansible_async
+
 
 Modules
 =======
@@ -98,7 +106,9 @@ Noteworthy module changes
 Plugins
 =======
 
-No notable changes.
+* The ``powershell`` shell plugin now uses ``async_dir`` to define the async path for the results file and the default
+  has changed to ``%USERPROFILE%\.ansible_async``. To control this path now, either set the ``ansible_async_dir``
+  variable or the ``async_dir`` value in the ``powershell`` section of the config ini.
 
 Porting custom scripts
 ======================

lib/ansible/executor/task_executor.py

@@ -751,8 +751,8 @@ class TaskExecutor:
         # Because this is an async task, the action handler is async. However,
         # we need the 'normal' action handler for the status check, so get it
         # now via the action_loader
-        normal_handler = self._shared_loader_obj.action_loader.get(
-            'normal',
+        async_handler = self._shared_loader_obj.action_loader.get(
+            'async_status',
             task=async_task,
             connection=self._connection,
             play_context=self._play_context,
@@ -766,7 +766,7 @@ class TaskExecutor:
             time.sleep(self._task.poll)
 
             try:
-                async_result = normal_handler.run(task_vars=task_vars)
+                async_result = async_handler.run(task_vars=task_vars)
                 # We do not bail out of the loop in cases where the failure
                 # is associated with a parsing error. The async_runner can
                 # have issues which result in a half-written/unparseable result
@@ -783,7 +783,7 @@ class TaskExecutor:
                 display.vvvv("Exception during async poll, retrying... (%s)" % to_text(e))
                 display.debug("Async poll exception was:\n%s" % to_text(traceback.format_exc()))
                 try:
-                    normal_handler._connection.reset()
+                    async_handler._connection.reset()
                 except AttributeError:
                     pass
 

lib/ansible/modules/utilities/logic/async_status.py

@@ -51,12 +51,13 @@ def main():
     module = AnsibleModule(argument_spec=dict(
         jid=dict(required=True),
         mode=dict(default='status', choices=['status', 'cleanup']),
+        # passed in from the async_status action plugin
+        _async_dir=dict(required=True, type='path'),
     ))
 
     mode = module.params['mode']
     jid = module.params['jid']
-
-    async_dir = os.environ.get('ANSIBLE_ASYNC_DIR', '~/.ansible_async')
+    async_dir = module.params['_async_dir']
 
     # setup logging directory
     logdir = os.path.expanduser(async_dir)

lib/ansible/modules/windows/async_status.ps1

@@ -9,13 +9,15 @@ $results = @{changed=$false}
 $parsed_args = Parse-Args $args
 $jid = Get-AnsibleParam $parsed_args "jid" -failifempty $true -resultobj $results
 $mode = Get-AnsibleParam $parsed_args "mode" -Default "status" -ValidateSet "status","cleanup"
-$_remote_tmp = Get-AnsibleParam $parsed_args "_ansible_remote_tmp" -type "path" -default $env:TMP
 
-$log_path = [System.IO.Path]::Combine($_remote_tmp, ".ansible_async", $jid)
+# parsed in from the async_status action plugin
+$async_dir = Get-AnsibleParam $parsed_args "_async_dir" -type "path" -failifempty $true
+
+$log_path = [System.IO.Path]::Combine($async_dir, $jid)
 
 If(-not $(Test-Path $log_path))
 {
-    Fail-Json @{ansible_job_id=$jid; started=1; finished=1} "could not find job"
+    Fail-Json @{ansible_job_id=$jid; started=1; finished=1} "could not find job at '$async_dir'"
 }
 
 If($mode -eq "cleanup") {

lib/ansible/plugins/action/__init__.py

@@ -734,6 +734,30 @@ class ActionBase(with_metaclass(ABCMeta, object)):
 
         self._update_module_args(module_name, module_args, task_vars)
 
+        # FIXME: convert async_wrapper.py to not rely on environment variables
+        # make sure we get the right async_dir variable, backwards compatibility
+        # means we need to lookup the env value ANSIBLE_ASYNC_DIR first
+        remove_async_dir = None
+        if wrap_async or self._task.async_val:
+            env_async_dir = [e for e in self._task.environment if
+                             "ANSIBLE_ASYNC_DIR" in e]
+            if len(env_async_dir) > 0:
+                msg = "Setting the async dir from the environment keyword " \
+                      "ANSIBLE_ASYNC_DIR is deprecated. Set the async_dir " \
+                      "shell option instead"
+                self._display.deprecated(msg, "2.12")
+            else:
+                # ANSIBLE_ASYNC_DIR is not set on the task, we get the value
+                # from the shell option and temporarily add to the environment
+                # list for async_wrapper to pick up
+                try:
+                    async_dir = self._connection._shell.get_option('async_dir')
+                except KeyError:
+                    # in case 3rd party plugin has not set this, use the default
+                    async_dir = "~/.ansible_async"
+                remove_async_dir = len(self._task.environment)
+                self._task.environment.append({"ANSIBLE_ASYNC_DIR": async_dir})
+
         # FUTURE: refactor this along with module build process to better encapsulate "smart wrapper" functionality
         (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
         display.vvv("Using module file %s" % module_path)
@@ -776,6 +800,12 @@ class ActionBase(with_metaclass(ABCMeta, object)):
 
         environment_string = self._compute_environment_string()
 
+        # remove the ANSIBLE_ASYNC_DIR env entry if we added a temporary one for
+        # the async_wrapper task - this is so the async_status plugin doesn't
+        # fire a deprecation warning when it runs after this task
+        if remove_async_dir is not None:
+            del self._task.environment[remove_async_dir]
+
         remote_files = []
         if tmpdir and remote_module_path:
             remote_files = [tmpdir, remote_module_path]

lib/ansible/plugins/action/async_status.py

@@ -0,0 +1,51 @@
+# Copyright: (c) 2018, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
+from ansible.utils.vars import merge_hash
+
+
+class ActionModule(ActionBase):
+
+    _VALID_ARGS = frozenset(('jid', 'mode'))
+
+    def run(self, tmp=None, task_vars=None):
+        results = super(ActionModule, self).run(tmp, task_vars)
+        del tmp  # tmp no longer has any effect
+
+        if "jid" not in self._task.args:
+            raise AnsibleError("jid is required")
+        jid = self._task.args["jid"]
+        mode = self._task.args.get("mode", "status")
+
+        env_async_dir = [e for e in self._task.environment if
+                         "ANSIBLE_ASYNC_DIR" in e]
+        if len(env_async_dir) > 0:
+            # for backwards compatibility we need to get the dir from
+            # ANSIBLE_ASYNC_DIR that is defined in the environment. This is
+            # deprecated and will be removed in favour of shell options
+            async_dir = env_async_dir[0]['ANSIBLE_ASYNC_DIR']
+
+            msg = "Setting the async dir from the environment keyword " \
+                  "ANSIBLE_ASYNC_DIR is deprecated. Set the async_dir " \
+                  "shell option instead"
+            self._display.deprecated(msg, "2.12")
+        else:
+            # inject the async directory based on the shell option into the
+            # module args
+            try:
+                async_dir = self._connection._shell.get_option('async_dir')
+            except KeyError:
+                # here for 3rd party shell plugin compatibility in case they do
+                # not define the async_dir option
+                async_dir = "~/.ansible_async"
+
+        module_args = dict(jid=jid, mode=mode, _async_dir=async_dir)
+        status = self._execute_module(task_vars=task_vars,
+                                      module_args=module_args)
+        results = merge_hash(results, status)
+        return results

lib/ansible/plugins/shell/powershell.py

@@ -12,6 +12,17 @@ DOCUMENTATION = '''
     description:
       - The only option when using 'winrm' as a connection plugin
     options:
+      async_dir:
+        description:
+        - Directory in which ansible will keep async job information.
+        - Before Ansible 2.8, this was set to C(remote_tmp + "\\.ansible_async").
+        default: '%USERPROFILE%\\.ansible_async'
+        ini:
+        - section: powershell
+          key: async_dir
+        vars:
+        - name: ansible_async_dir
+        version_added: '2.8'
       remote_tmp:
         description:
         - Temporary directory to use on targets when copying files to the host.
@@ -1213,14 +1224,18 @@ $exec_wrapper = {
 
 
 Function Run($payload) {
-    $remote_tmp = $payload["module_args"]["_ansible_remote_tmp"]
-    $remote_tmp = [System.Environment]::ExpandEnvironmentVariables($remote_tmp)
+    if ($payload.environment.ContainsKey("ANSIBLE_ASYNC_DIR")) {
+        $async_dir = $payload.environment.ANSIBLE_ASYNC_DIR
+    } else {
+        $async_dir = "%USERPROFILE%\.ansible_async"
+    }
+    $async_dir = [System.Environment]::ExpandEnvironmentVariables($async_dir)
 
     # calculate the result path so we can include it in the worker payload
     $jid = $payload.async_jid
     $local_jid = $jid + "." + $pid
 
-    $results_path = [System.IO.Path]::Combine($remote_tmp, ".ansible_async", $local_jid)
+    $results_path = [System.IO.Path]::Combine($async_dir, $local_jid)
 
     $payload.async_results_path = $results_path
 

lib/ansible/utils/module_docs_fragments/shell_common.py

@@ -32,7 +32,7 @@ options:
       - name: ansible_system_tmpdirs
   async_dir:
     description:
-       - Directory in which ansible will keep async job inforamtion
+       - Directory in which ansible will keep async job information
     default: '~/.ansible_async'
     env: [{name: ANSIBLE_ASYNC_DIR}]
     ini:

test/integration/targets/async/tasks/main.yml

@@ -177,3 +177,108 @@
     - non_async_result is changed
     - non_async_result is finished
     - "'ansible_job_id' not in non_async_result"
+
+- name: set fact of custom tmp dir
+  set_fact:
+    custom_async_tmp: ~/.ansible_async_test
+
+- name: ensure custom async tmp dir is absent
+  file:
+    path: '{{ custom_async_tmp }}'
+    state: absent
+
+- block:
+  - name: run async task with custom dir
+    command: sleep 1
+    register: async_custom_dir
+    async: 5
+    poll: 1
+    vars:
+      ansible_async_dir: '{{ custom_async_tmp }}'
+
+  - name: check if the async temp dir is created
+    stat:
+      path: '{{ custom_async_tmp }}'
+    register: async_custom_dir_result
+
+  - name: assert run async task with custom dir
+    assert:
+      that:
+      - async_custom_dir is successful
+      - async_custom_dir is finished
+      - async_custom_dir_result.stat.exists
+
+  - name: remove custom async dir again
+    file:
+      path: '{{ custom_async_tmp }}'
+      state: absent
+
+  - name: run async task with custom dir - deprecated format
+    command: sleep 1
+    register: async_custom_dir_dep
+    async: 5
+    poll: 1
+    environment:
+      ANSIBLE_ASYNC_DIR: '{{ custom_async_tmp }}'
+
+  - name: check if the async temp dir is created - deprecated format
+    stat:
+      path: '{{ custom_async_tmp }}'
+    register: async_custom_dir_dep_result
+
+  - name: assert run async task with custom dir - deprecated format
+    assert:
+      that:
+      - async_custom_dir_dep is successful
+      - async_custom_dir_dep is finished
+      - async_custom_dir_dep_result.stat.exists
+
+  - name: remove custom async dir after deprecation test
+    file:
+      path: '{{ custom_async_tmp }}'
+      state: absent
+
+  - name: run fire and forget async task with custom dir
+    command: sleep 1
+    register: async_fandf_custom_dir
+    async: 5
+    poll: 0
+    vars:
+      ansible_async_dir: '{{ custom_async_tmp }}'
+
+  - name: fail to get async status with custom dir with defaults
+    async_status:
+      jid: '{{ async_fandf_custom_dir.ansible_job_id }}'
+    register: async_fandf_custom_dir_fail
+    ignore_errors: yes
+
+  - name: get async status with custom dir using newer format
+    async_status:
+      jid: '{{ async_fandf_custom_dir.ansible_job_id }}'
+    register: async_fandf_custom_dir_result
+    vars:
+      ansible_async_dir: '{{ custom_async_tmp }}'
+
+  - name: get async status with custom dir - deprecated format
+    async_status:
+      jid: '{{ async_fandf_custom_dir.ansible_job_id }}'
+    register: async_fandf_custom_dir_dep_result
+    environment:
+      ANSIBLE_ASYNC_DIR: '{{ custom_async_tmp }}'
+
+  - name: assert run fire and forget async task with custom dir
+    assert:
+      that:
+      - async_fandf_custom_dir is successful
+      - async_fandf_custom_dir_fail is failed
+      - async_fandf_custom_dir_fail.msg == "could not find job"
+      - async_fandf_custom_dir_result is successful
+      - async_fandf_custom_dir_result is finished
+      - async_fandf_custom_dir_dep_result is successful
+      - async_fandf_custom_dir_dep_result is finished
+
+  always:
+  - name: remove custom tmp dir after test
+    file:
+      path: '{{ custom_async_tmp }}'
+      state: absent

test/integration/targets/win_async_wrapper/tasks/main.yml

@@ -166,17 +166,46 @@
     - nonascii_output.stdout_lines[0] == 'über den Fußgängerübergang gehen'
     - nonascii_output.stderr == ''
 
-- name: test async with custom remote_tmp
+- name: test async with custom async dir
   win_shell: echo hi
-  register: async_custom_tmp
+  register: async_custom_dir
   async: 5
   vars:
-    ansible_remote_tmp: '{{win_output_dir}}'
+    ansible_async_dir: '{{win_output_dir}}'
 
 - name: assert results file is in the remote tmp specified
   assert:
     that:
-    - async_custom_tmp.results_file == win_output_dir + '\\.ansible_async\\' + async_custom_tmp.ansible_job_id
+    - async_custom_dir.results_file == win_output_dir + '\\' + async_custom_dir.ansible_job_id
+
+- name: test async fire and forget with custom async dir
+  win_shell: echo hi
+  register: async_custom_dir_poll
+  async: 5
+  poll: 0
+  vars:
+    ansible_async_dir: '{{win_output_dir}}'
+
+- name: poll with different dir - fail
+  async_status:
+    jid: '{{ async_custom_dir_poll.ansible_job_id }}'
+  register: fail_async_custom_dir_poll
+  ignore_errors: yes
+
+- name: poll with different dir - success
+  async_status:
+    jid: '{{ async_custom_dir_poll.ansible_job_id }}'
+  register: success_async_custom_dir_poll
+  vars:
+    ansible_async_dir: '{{win_output_dir}}'
+
+- name: assert test async fire and forget with custom async dir
+  assert:
+    that:
+    - fail_async_custom_dir_poll.failed
+    - '"could not find job at ''" + nonascii_output.results_file|win_dirname + "''" in fail_async_custom_dir_poll.msg'
+    - not success_async_custom_dir_poll.failed
+    - success_async_custom_dir_poll.results_file == win_output_dir + '\\' + async_custom_dir_poll.ansible_job_id
 
 # FUTURE: figure out why the last iteration of this test often fails on shippable
 #- name: loop async success