added toggle to control vars plugin behviour

this should allow user to control how they want the playbook dirs inspected
for additional vars, default now reverts to 2.3 behaviour (top).
corrected paths order
minor doc reword

docs/docsite/rst/porting_guide_2.4.rst

@@ -99,7 +99,10 @@ There have been many changes to the implementation of vars plugins, but both use
 
 The most notable difference to users is that vars plugins now get invoked on demand instead of at inventory build time.  This should make them more efficient for large inventories, especially when using a subset of the hosts.
 
-.. note:: This also creates a difference with group/host_vars when using them adjacent to playbooks. Before, the 'first' playbook loaded determined the variables; now the 'current' playbook does. We are looking to fix this soon, since 'all playbooks' in the path should be considered for variable loading.
+
+.. note::
+  - This also creates a difference with group/host_vars when using them adjacent to playbooks. Before, the 'first' playbook loaded determined the variables; now the 'current' playbook does. We are looking to fix this soon, since 'all playbooks' in the path should be considered for variable loading.
+  - In 2.4.1 we added a toggle to allow you to control this behaviour, 'top' will be the pre 2.4, 'bottom' will use the current playbook hosting the task and 'all' will use them all from top to bottom.
 
 
 Inventory plugins

lib/ansible/config/base.yml

@@ -1384,6 +1384,19 @@ PERSISTENT_COMMAND_TIMEOUT:
   ini:
   - {key: command_timeout, section: persistent_connection}
   type: int
+PLAYBOOK_VARS_ROOT:
+  name: playbook vars files root
+  default: top
+  version_added: "2.4.1"
+  description:
+    - This sets which playbook dirs will be used as a root to process vars plugins, which includes finding host_vars/group_vars
+    - The ``top`` option follows the traditional behaviour of using the top playbook in the chain to find the root directory.
+    - The ``bottom`` option follows the 2.4.0 behaviour of using the current playbook to find the root directory.
+    - The ``all`` option examines from the first parent to the current playbook.
+  env: [{name: ANSIBLE_PLAYBOOK_VARS_ROOT}]
+  ini:
+  - {key: playbook_vars_root, section: defaults}
+  choices: [ top, bottom, all ]
 RETRY_FILES_ENABLED:
   name: Retry files
   default: True

lib/ansible/vars/manager.py

@@ -234,16 +234,26 @@ class VariableManager:
             for role in play.get_roles():
                 all_vars = combine_vars(all_vars, role.get_default_vars())
 
-        # if we have a task in this context, and that task has a role, make
-        # sure it sees its defaults above any other roles, as we previously
-        # (v1) made sure each task had a copy of its roles default vars
-        if task and task._role is not None and (play or task.action == 'include_role'):
-            all_vars = combine_vars(all_vars, task._role.get_default_vars(dep_chain=task.get_dep_chain()))
+        basedirs = []
+        if task:
+            # set basedirs
+            if C.PLAYBOOK_VARS_ROOT == 'all':  # should be default
+                basedirs = task.get_search_path()
+            elif C.PLAYBOOK_VARS_ROOT == 'top':  # only option pre 2.3
+                basedirs = [self._loader.get_basedir()]
+            elif C.PLAYBOOK_VARS_ROOT in ('bottom', 'playbook_dir'):  # only option in 2.4.0
+                basedirs = [task.get_search_path()[0]]
+            else:
+                raise AnsibleError('Unkown playbook vars logic: %s' % C.PLAYBOOK_VARS_ROOT)
+
+            # if we have a task in this context, and that task has a role, make
+            # sure it sees its defaults above any other roles, as we previously
+            # (v1) made sure each task had a copy of its roles default vars
+            if task._role is not None and (play or task.action == 'include_role'):
+                all_vars = combine_vars(all_vars, task._role.get_default_vars(dep_chain=task.get_dep_chain()))
 
         if host:
-            # INIT WORK (use unsafe as we are going to copy/merge vars, no need to x2 copy)
-            # basedir, THE 'all' group and the rest of groups for a host, used below
-            basedir = self._loader.get_basedir()
+            # THE 'all' group and the rest of groups for a host, used below
             all_group = self._inventory.groups.get('all')
             host_groups = sort_groups([g for g in host.get_groups() if g.name not in ['all']])
 
@@ -283,7 +293,8 @@ class VariableManager:
                 ''' merges all entities adjacent to play '''
                 data = {}
                 for plugin in vars_loader.all():
-                    data = combine_vars(data, _get_plugin_vars(plugin, basedir, entities))
+                    for path in basedirs:
+                        data = combine_vars(data, _get_plugin_vars(plugin, path, entities))
                 return data
 
             # configurable functions that are sortable via config, rememer to add to _ALLOWED if expanding this list