ensure inventory plugin loading rel to play (#51177)

Ensure inventory plugin loading rel to play

  fixes #51033

*  clarify paths
* now adding dirs funciton in loader
* better warnings
* each cli should handle adding dirs depending on context

changelogs/fragments/allow_inv_plugin_loading.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - allow loading inventory plugins adjacent to playbooks

lib/ansible/cli/__init__.py

@@ -9,7 +9,6 @@ __metaclass__ = type
 
 import getpass
 import os
-import os.path
 import re
 import subprocess
 import sys

lib/ansible/plugins/loader.py

@@ -51,7 +51,7 @@ def add_all_plugin_dirs(path):
                 if os.path.isdir(plugin_path):
                     obj.add_directory(to_text(plugin_path))
     else:
-        display.warning("Ignoring invalid path provided to plugin path: %s is not a directory" % to_native(path))
+        display.warning("Ignoring invalid path provided to plugin path: '%s' is not a directory" % to_native(path))
 
 
 def get_shell_plugin(shell_type=None, executable=None):
@@ -87,6 +87,13 @@ def get_shell_plugin(shell_type=None, executable=None):
     return shell
 
 
+def add_dirs_to_loader(which_loader, paths):
+
+    loader = getattr(sys.modules[__name__], '%s_loader' % which_loader)
+    for path in paths:
+        loader.add_directory(path, with_subdir=True)
+
+
 class PluginLoader:
     '''
     PluginLoader loads plugins from the configured plugin directories.
@@ -435,6 +442,7 @@ class PluginLoader:
         #       looks like _get_paths() never forces a cache refresh so if we expect
         #       additional directories to be added later, it is buggy.
         for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)):
+            display.debug('trying %s' % path)
             try:
                 full_paths = (os.path.join(path, f) for f in os.listdir(path))
             except OSError as e:

test/integration/targets/rel_plugin_loading/aliases

@@ -0,0 +1 @@
+shippable/posix/group3

test/integration/targets/rel_plugin_loading/notyaml.yml

@@ -0,0 +1,5 @@
+all:
+  hosts:
+    testhost:
+        ansible_connection: local
+        ansible_python_interpreter: "{{ansible_playbook_python}}"

test/integration/targets/rel_plugin_loading/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ANSIBLE_INVENTORY_ENABLED=notyaml ansible-playbook subdir/play.yml -i notyaml.yml "$@"

test/integration/targets/rel_plugin_loading/subdir/inventory_plugins/notyaml.py

@@ -0,0 +1,168 @@
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    inventory: yaml
+    version_added: "2.4"
+    short_description: Uses a specific YAML file as an inventory source.
+    description:
+        - "YAML-based inventory, should start with the C(all) group and contain hosts/vars/children entries."
+        - Host entries can have sub-entries defined, which will be treated as variables.
+        - Vars entries are normal group vars.
+        - "Children are 'child groups', which can also have their own vars/hosts/children and so on."
+        - File MUST have a valid extension, defined in configuration.
+    notes:
+        - If you want to set vars for the C(all) group inside the inventory file, the C(all) group must be the first entry in the file.
+        - Whitelisted in configuration by default.
+    options:
+      yaml_extensions:
+        description: list of 'valid' extensions for files containing YAML
+        type: list
+        default: ['.yaml', '.yml', '.json']
+        env:
+          - name: ANSIBLE_YAML_FILENAME_EXT
+          - name: ANSIBLE_INVENTORY_PLUGIN_EXTS
+        ini:
+          - key: yaml_valid_extensions
+            section: defaults
+          - section: inventory_plugin_yaml
+            key: yaml_valid_extensions
+
+'''
+EXAMPLES = '''
+all: # keys must be unique, i.e. only one 'hosts' per group
+    hosts:
+        test1:
+        test2:
+            host_var: value
+    vars:
+        group_all_var: value
+    children:   # key order does not matter, indentation does
+        other_group:
+            children:
+                group_x:
+                    hosts:
+                        test5
+            vars:
+                g2_var2: value3
+            hosts:
+                test4:
+                    ansible_host: 127.0.0.1
+        last_group:
+            hosts:
+                test1 # same host as above, additional group membership
+            vars:
+                group_last_var: value
+'''
+
+import os
+
+from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.module_utils.six import string_types
+from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import MutableMapping
+from ansible.plugins.inventory import BaseFileInventoryPlugin
+
+NoneType = type(None)
+
+
+class InventoryModule(BaseFileInventoryPlugin):
+
+    NAME = 'yaml'
+
+    def __init__(self):
+
+        super(InventoryModule, self).__init__()
+
+    def verify_file(self, path):
+
+        valid = False
+        if super(InventoryModule, self).verify_file(path):
+            file_name, ext = os.path.splitext(path)
+            if not ext or ext in self.get_option('yaml_extensions'):
+                valid = True
+        return valid
+
+    def parse(self, inventory, loader, path, cache=True):
+        ''' parses the inventory file '''
+
+        super(InventoryModule, self).parse(inventory, loader, path)
+        self.set_options()
+
+        try:
+            data = self.loader.load_from_file(path, cache=False)
+        except Exception as e:
+            raise AnsibleParserError(e)
+
+        if not data:
+            raise AnsibleParserError('Parsed empty YAML file')
+        elif not isinstance(data, MutableMapping):
+            raise AnsibleParserError('YAML inventory has invalid structure, it should be a dictionary, got: %s' % type(data))
+        elif data.get('plugin'):
+            raise AnsibleParserError('Plugin configuration YAML file, not YAML inventory')
+
+        # We expect top level keys to correspond to groups, iterate over them
+        # to get host, vars and subgroups (which we iterate over recursivelly)
+        if isinstance(data, MutableMapping):
+            for group_name in data:
+                self._parse_group(group_name, data[group_name])
+        else:
+            raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(data))
+
+    def _parse_group(self, group, group_data):
+
+        if isinstance(group_data, (MutableMapping, NoneType)):
+
+            try:
+                self.inventory.add_group(group)
+            except AnsibleError as e:
+                raise AnsibleParserError("Unable to add group %s: %s" % (group, to_text(e)))
+
+            if group_data is not None:
+                # make sure they are dicts
+                for section in ['vars', 'children', 'hosts']:
+                    if section in group_data:
+                        # convert strings to dicts as these are allowed
+                        if isinstance(group_data[section], string_types):
+                            group_data[section] = {group_data[section]: None}
+
+                        if not isinstance(group_data[section], (MutableMapping, NoneType)):
+                            raise AnsibleParserError('Invalid "%s" entry for "%s" group, requires a dictionary, found "%s" instead.' %
+                                                     (section, group, type(group_data[section])))
+
+                for key in group_data:
+
+                    if not isinstance(group_data[key], (MutableMapping, NoneType)):
+                        self.display.warning('Skipping key (%s) in group (%s) as it is not a mapping, it is a %s' % (key, group, type(group_data[key])))
+                        continue
+
+                    if isinstance(group_data[key], NoneType):
+                        self.display.vvv('Skipping empty key (%s) in group (%s)' % (key, group))
+                    elif key == 'vars':
+                        for var in group_data[key]:
+                            self.inventory.set_variable(group, var, group_data[key][var])
+                    elif key == 'children':
+                        for subgroup in group_data[key]:
+                            self._parse_group(subgroup, group_data[key][subgroup])
+                            self.inventory.add_child(group, subgroup)
+
+                    elif key == 'hosts':
+                        for host_pattern in group_data[key]:
+                            hosts, port = self._parse_host(host_pattern)
+                            self._populate_host_vars(hosts, group_data[key][host_pattern] or {}, group, port)
+                    else:
+                        self.display.warning('Skipping unexpected key (%s) in group (%s), only "vars", "children" and "hosts" are valid' % (key, group))
+
+        else:
+            self.display.warning("Skipping '%s' as this is not a valid group definition" % group)
+
+    def _parse_host(self, host_pattern):
+        '''
+        Each host key can be a pattern, try to process it and add variables as needed
+        '''
+        (hostnames, port) = self._expand_hostpattern(host_pattern)
+
+        return hostnames, port

test/integration/targets/rel_plugin_loading/subdir/play.yml

@@ -0,0 +1,6 @@
+- hosts: all
+  gather_facts: false
+  tasks:
+    - assert:
+        that:
+            - inventory_hostname == 'testhost'