Break apart a looped dependency to show a warning when parsing playbooks

Display a warning when a dict key is overwritten by pyyaml
Fixes #12888

hacking/test-module

@@ -36,7 +36,7 @@ import subprocess
 import traceback
 import optparse
 import ansible.utils.vars as utils_vars
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.utils.jsonify import jsonify
 from ansible.parsing.splitter import parse_kv
 import ansible.executor.module_common as module_common

lib/ansible/cli/adhoc.py

@@ -27,7 +27,7 @@ from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.inventory import Inventory
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
 from ansible.plugins import get_all_plugin_loaders

lib/ansible/cli/playbook.py

@@ -29,7 +29,7 @@ from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.playbook_executor import PlaybookExecutor
 from ansible.inventory import Inventory
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.utils.vars import load_extra_vars
 from ansible.vars import VariableManager
 

lib/ansible/cli/vault.py

@@ -21,14 +21,11 @@ __metaclass__ = type
 
 import os
 import sys
-import traceback
 
-from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.vault import VaultEditor
 from ansible.cli import CLI
-from ansible.utils.display import Display
 
 class VaultCLI(CLI):
     """ Vault command line class """

lib/ansible/constants.py

@@ -25,7 +25,7 @@ from string import ascii_letters, digits
 from ansible.compat.six import string_types
 from ansible.compat.six.moves import configparser
 
-from ansible.parsing.splitter import unquote
+from ansible.parsing.quoting import unquote
 from ansible.errors import AnsibleOptionsError
 
 # copied from utils, avoid circular reference fun :)

lib/ansible/executor/task_result.py

@@ -19,7 +19,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 
 class TaskResult:
     '''

lib/ansible/parsing/__init__.py

@@ -1,4 +1,4 @@
-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2015, Toshio Kuratomi <tkuratomi@ansbile.com>
 #
 # This file is part of Ansible
 #
@@ -18,278 +18,3 @@
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
-
-import copy
-import json
-import os
-import stat
-import subprocess
-
-from yaml import load, YAMLError
-from ansible.compat.six import text_type, string_types
-
-from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
-from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
-from ansible.parsing.vault import VaultLib
-from ansible.parsing.splitter import unquote
-from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
-from ansible.module_utils.basic import is_executable
-from ansible.utils.path import unfrackpath
-from ansible.utils.unicode import to_unicode
-
-class DataLoader():
-
-    '''
-    The DataLoader class is used to load and parse YAML or JSON content,
-    either from a given file name or from a string that was previously
-    read in through other means. A Vault password can be specified, and
-    any vault-encrypted files will be decrypted.
-
-    Data read from files will also be cached, so the file will never be
-    read from disk more than once.
-
-    Usage:
-
-        dl = DataLoader()
-        (or)
-        dl = DataLoader(vault_password='foo')
-
-        ds = dl.load('...')
-        ds = dl.load_from_file('/path/to/file')
-    '''
-
-    def __init__(self):
-        self._basedir = '.'
-        self._FILE_CACHE = dict()
-
-        # initialize the vault stuff with an empty password
-        self.set_vault_password(None)
-
-    def set_vault_password(self, vault_password):
-        self._vault_password = vault_password
-        self._vault = VaultLib(password=vault_password)
-
-    def load(self, data, file_name='<string>', show_content=True):
-        '''
-        Creates a python datastructure from the given data, which can be either
-        a JSON or YAML string. 
-        '''
-
-        try:
-            # we first try to load this data as JSON
-            return json.loads(data)
-        except:
-            # if loading JSON failed for any reason, we go ahead
-            # and try to parse it as YAML instead
-
-            if isinstance(data, AnsibleUnicode):
-                # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
-                # they are unable to cope with our subclass.
-                # Unwrap and re-wrap the unicode so we can keep track of line
-                # numbers
-                new_data = text_type(data)
-            else:
-                new_data = data
-            try:
-                new_data = self._safe_load(new_data, file_name=file_name)
-            except YAMLError as yaml_exc:
-                self._handle_error(yaml_exc, file_name, show_content)
-
-            if isinstance(data, AnsibleUnicode):
-                new_data = AnsibleUnicode(new_data)
-                new_data.ansible_pos = data.ansible_pos
-            return new_data
-
-    def load_from_file(self, file_name):
-        ''' Loads data from a file, which can contain either JSON or YAML.  '''
-
-        file_name = self.path_dwim(file_name)
-
-        # if the file has already been read in and cached, we'll
-        # return those results to avoid more file/vault operations
-        if file_name in self._FILE_CACHE:
-            parsed_data = self._FILE_CACHE[file_name]
-        else:
-            # read the file contents and load the data structure from them
-            (file_data, show_content) = self._get_file_contents(file_name)
-            parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content)
-
-            # cache the file contents for next time
-            self._FILE_CACHE[file_name] = parsed_data
-
-        # return a deep copy here, so the cache is not affected
-        return copy.deepcopy(parsed_data)
-
-    def path_exists(self, path):
-        path = self.path_dwim(path)
-        return os.path.exists(path)
-
-    def is_file(self, path):
-        path = self.path_dwim(path)
-        return os.path.isfile(path) or path == os.devnull
-
-    def is_directory(self, path):
-        path = self.path_dwim(path)
-        return os.path.isdir(path)
-
-    def list_directory(self, path):
-        path = self.path_dwim(path)
-        return os.listdir(path)
-
-    def is_executable(self, path):
-        '''is the given path executable?'''
-        path = self.path_dwim(path)
-        return is_executable(path)
-
-    def _safe_load(self, stream, file_name=None):
-        ''' Implements yaml.safe_load(), except using our custom loader class. '''
-
-        loader = AnsibleLoader(stream, file_name)
-        try:
-            return loader.get_single_data()
-        finally:
-            loader.dispose()
-
-    def _get_file_contents(self, file_name):
-        '''
-        Reads the file contents from the given file name, and will decrypt them
-        if they are found to be vault-encrypted.
-        '''
-        if not file_name or not isinstance(file_name, string_types):
-            raise AnsibleParserError("Invalid filename: '%s'" % str(file_name))
-
-        if not self.path_exists(file_name) or not self.is_file(file_name):
-            raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name)
-
-        show_content = True
-        try:
-            with open(file_name, 'rb') as f:
-                data = f.read()
-                if self._vault.is_encrypted(data):
-                    data = self._vault.decrypt(data)
-                    show_content = False
-
-            data = to_unicode(data, errors='strict')
-            return (data, show_content)
-
-        except (IOError, OSError) as e:
-            raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)))
-
-    def _handle_error(self, yaml_exc, file_name, show_content):
-        '''
-        Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
-        file name/position where a YAML exception occurred, and raises an AnsibleParserError
-        to display the syntax exception information.
-        '''
-
-        # if the YAML exception contains a problem mark, use it to construct
-        # an object the error class can use to display the faulty line
-        err_obj = None
-        if hasattr(yaml_exc, 'problem_mark'):
-            err_obj = AnsibleBaseYAMLObject()
-            err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
-
-        raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content)
-
-    def get_basedir(self):
-        ''' returns the current basedir '''
-        return self._basedir
-
-    def set_basedir(self, basedir):
-        ''' sets the base directory, used to find files when a relative path is given '''
-
-        if basedir is not None:
-            self._basedir = to_unicode(basedir)
-
-    def path_dwim(self, given):
-        '''
-        make relative paths work like folks expect.
-        '''
-
-        given = unquote(given)
-
-        if given.startswith("/"):
-            return os.path.abspath(given)
-        elif given.startswith("~"):
-            return os.path.abspath(os.path.expanduser(given))
-        else:
-            return os.path.abspath(os.path.join(self._basedir, given))
-
-    def path_dwim_relative(self, path, dirname, source):
-        '''
-        find one file in either a role or playbook dir with or without
-        explicitly named dirname subdirs
-
-        Used in action plugins and lookups to find supplemental files that
-        could be in either place.
-        '''
-
-        search = []
-        isrole = False
-
-        # I have full path, nothing else needs to be looked at
-        if source.startswith('~') or source.startswith('/'):
-            search.append(self.path_dwim(source))
-        else:
-            # base role/play path + templates/files/vars + relative filename
-            search.append(os.path.join(path, dirname, source))
-
-            basedir = unfrackpath(path)
-
-            # is it a role and if so make sure you get correct base path
-            if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \
-                or os.path.exists(os.path.join(path,'tasks/main.yml')):
-                isrole = True
-                if path.endswith('tasks'):
-                    basedir = unfrackpath(os.path.dirname(path))
-
-            cur_basedir = self._basedir
-            self.set_basedir(basedir)
-            # resolved base role/play path + templates/files/vars + relative filename
-            search.append(self.path_dwim(os.path.join(basedir, dirname, source)))
-            self.set_basedir(cur_basedir)
-
-            if isrole and not source.endswith(dirname):
-                # look in role's tasks dir w/o dirname
-                search.append(self.path_dwim(os.path.join(basedir, 'tasks', source)))
-
-            # try to create absolute path for loader basedir + templates/files/vars + filename
-            search.append(self.path_dwim(os.path.join(dirname,source)))
-            search.append(self.path_dwim(os.path.join(basedir, source)))
-
-            # try to create absolute path for loader basedir + filename
-            search.append(self.path_dwim(source))
-
-        for candidate in search:
-            if os.path.exists(candidate):
-                break
-
-        return candidate
-
-    def read_vault_password_file(self, vault_password_file):
-        """
-        Read a vault password from a file or if executable, execute the script and
-        retrieve password from STDOUT
-        """
-
-        this_path = os.path.realpath(os.path.expanduser(vault_password_file))
-        if not os.path.exists(this_path):
-            raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
-
-        if self.is_executable(this_path):
-            try:
-                # STDERR not captured to make it easier for users to prompt for input in their scripts
-                p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
-            except OSError as e:
-                raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
-            stdout, stderr = p.communicate()
-            self.set_vault_password(stdout.strip('\r\n'))
-        else:
-            try:
-                f = open(this_path, "rb")
-                self.set_vault_password(f.read().strip())
-                f.close()
-            except (OSError, IOError) as e:
-                raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))
-

lib/ansible/parsing/dataloader.py

@@ -0,0 +1,294 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import copy
+import json
+import os
+import stat
+import subprocess
+
+from yaml import load, YAMLError
+from ansible.compat.six import text_type, string_types
+
+from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
+from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
+from ansible.parsing.vault import VaultLib
+from ansible.parsing.quoting import unquote
+from ansible.parsing.yaml.loader import AnsibleLoader
+from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
+from ansible.module_utils.basic import is_executable
+from ansible.utils.path import unfrackpath
+from ansible.utils.unicode import to_unicode
+
+class DataLoader():
+
+    '''
+    The DataLoader class is used to load and parse YAML or JSON content,
+    either from a given file name or from a string that was previously
+    read in through other means. A Vault password can be specified, and
+    any vault-encrypted files will be decrypted.
+
+    Data read from files will also be cached, so the file will never be
+    read from disk more than once.
+
+    Usage:
+
+        dl = DataLoader()
+        (or)
+        dl = DataLoader(vault_password='foo')
+
+        ds = dl.load('...')
+        ds = dl.load_from_file('/path/to/file')
+    '''
+
+    def __init__(self):
+        self._basedir = '.'
+        self._FILE_CACHE = dict()
+
+        # initialize the vault stuff with an empty password
+        self.set_vault_password(None)
+
+    def set_vault_password(self, vault_password):
+        self._vault_password = vault_password
+        self._vault = VaultLib(password=vault_password)
+
+    def load(self, data, file_name='<string>', show_content=True):
+        '''
+        Creates a python datastructure from the given data, which can be either
+        a JSON or YAML string.
+        '''
+
+        try:
+            # we first try to load this data as JSON
+            return json.loads(data)
+        except:
+            # if loading JSON failed for any reason, we go ahead
+            # and try to parse it as YAML instead
+
+            if isinstance(data, AnsibleUnicode):
+                # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
+                # they are unable to cope with our subclass.
+                # Unwrap and re-wrap the unicode so we can keep track of line
+                # numbers
+                new_data = text_type(data)
+            else:
+                new_data = data
+            try:
+                new_data = self._safe_load(new_data, file_name=file_name)
+            except YAMLError as yaml_exc:
+                self._handle_error(yaml_exc, file_name, show_content)
+
+            if isinstance(data, AnsibleUnicode):
+                new_data = AnsibleUnicode(new_data)
+                new_data.ansible_pos = data.ansible_pos
+            return new_data
+
+    def load_from_file(self, file_name):
+        ''' Loads data from a file, which can contain either JSON or YAML.  '''
+
+        file_name = self.path_dwim(file_name)
+
+        # if the file has already been read in and cached, we'll
+        # return those results to avoid more file/vault operations
+        if file_name in self._FILE_CACHE:
+            parsed_data = self._FILE_CACHE[file_name]
+        else:
+            # read the file contents and load the data structure from them
+            (file_data, show_content) = self._get_file_contents(file_name)
+            parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content)
+
+            # cache the file contents for next time
+            self._FILE_CACHE[file_name] = parsed_data
+
+        # return a deep copy here, so the cache is not affected
+        return copy.deepcopy(parsed_data)
+
+    def path_exists(self, path):
+        path = self.path_dwim(path)
+        return os.path.exists(path)
+
+    def is_file(self, path):
+        path = self.path_dwim(path)
+        return os.path.isfile(path) or path == os.devnull
+
+    def is_directory(self, path):
+        path = self.path_dwim(path)
+        return os.path.isdir(path)
+
+    def list_directory(self, path):
+        path = self.path_dwim(path)
+        return os.listdir(path)
+
+    def is_executable(self, path):
+        '''is the given path executable?'''
+        path = self.path_dwim(path)
+        return is_executable(path)
+
+    def _safe_load(self, stream, file_name=None):
+        ''' Implements yaml.safe_load(), except using our custom loader class. '''
+
+        loader = AnsibleLoader(stream, file_name)
+        try:
+            return loader.get_single_data()
+        finally:
+            loader.dispose()
+
+    def _get_file_contents(self, file_name):
+        '''
+        Reads the file contents from the given file name, and will decrypt them
+        if they are found to be vault-encrypted.
+        '''
+        if not file_name or not isinstance(file_name, string_types):
+            raise AnsibleParserError("Invalid filename: '%s'" % str(file_name))
+
+        if not self.path_exists(file_name) or not self.is_file(file_name):
+            raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name)
+
+        show_content = True
+        try:
+            with open(file_name, 'rb') as f:
+                data = f.read()
+                if self._vault.is_encrypted(data):
+                    data = self._vault.decrypt(data)
+                    show_content = False
+
+            data = to_unicode(data, errors='strict')
+            return (data, show_content)
+
+        except (IOError, OSError) as e:
+            raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)))
+
+    def _handle_error(self, yaml_exc, file_name, show_content):
+        '''
+        Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
+        file name/position where a YAML exception occurred, and raises an AnsibleParserError
+        to display the syntax exception information.
+        '''
+
+        # if the YAML exception contains a problem mark, use it to construct
+        # an object the error class can use to display the faulty line
+        err_obj = None
+        if hasattr(yaml_exc, 'problem_mark'):
+            err_obj = AnsibleBaseYAMLObject()
+            err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
+
+        raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content)
+
+    def get_basedir(self):
+        ''' returns the current basedir '''
+        return self._basedir
+
+    def set_basedir(self, basedir):
+        ''' sets the base directory, used to find files when a relative path is given '''
+
+        if basedir is not None:
+            self._basedir = to_unicode(basedir)
+
+    def path_dwim(self, given):
+        '''
+        make relative paths work like folks expect.
+        '''
+
+        given = unquote(given)
+
+        if given.startswith("/"):
+            return os.path.abspath(given)
+        elif given.startswith("~"):
+            return os.path.abspath(os.path.expanduser(given))
+        else:
+            return os.path.abspath(os.path.join(self._basedir, given))
+
+    def path_dwim_relative(self, path, dirname, source):
+        '''
+        find one file in either a role or playbook dir with or without
+        explicitly named dirname subdirs
+
+        Used in action plugins and lookups to find supplemental files that
+        could be in either place.
+        '''
+
+        search = []
+        isrole = False
+
+        # I have full path, nothing else needs to be looked at
+        if source.startswith('~') or source.startswith('/'):
+            search.append(self.path_dwim(source))
+        else:
+            # base role/play path + templates/files/vars + relative filename
+            search.append(os.path.join(path, dirname, source))
+
+            basedir = unfrackpath(path)
+
+            # is it a role and if so make sure you get correct base path
+            if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \
+                or os.path.exists(os.path.join(path,'tasks/main.yml')):
+                isrole = True
+                if path.endswith('tasks'):
+                    basedir = unfrackpath(os.path.dirname(path))
+
+            cur_basedir = self._basedir
+            self.set_basedir(basedir)
+            # resolved base role/play path + templates/files/vars + relative filename
+            search.append(self.path_dwim(os.path.join(basedir, dirname, source)))
+            self.set_basedir(cur_basedir)
+
+            if isrole and not source.endswith(dirname):
+                # look in role's tasks dir w/o dirname
+                search.append(self.path_dwim(os.path.join(basedir, 'tasks', source)))
+
+            # try to create absolute path for loader basedir + templates/files/vars + filename
+            search.append(self.path_dwim(os.path.join(dirname,source)))
+            search.append(self.path_dwim(os.path.join(basedir, source)))
+
+            # try to create absolute path for loader basedir + filename
+            search.append(self.path_dwim(source))
+
+        for candidate in search:
+            if os.path.exists(candidate):
+                break
+
+        return candidate
+
+    def read_vault_password_file(self, vault_password_file):
+        """
+        Read a vault password from a file or if executable, execute the script and
+        retrieve password from STDOUT
+        """
+
+        this_path = os.path.realpath(os.path.expanduser(vault_password_file))
+        if not os.path.exists(this_path):
+            raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
+
+        if self.is_executable(this_path):
+            try:
+                # STDERR not captured to make it easier for users to prompt for input in their scripts
+                p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
+            except OSError as e:
+                raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
+            stdout, stderr = p.communicate()
+            self.set_vault_password(stdout.strip('\r\n'))
+        else:
+            try:
+                f = open(this_path, "rb")
+                self.set_vault_password(f.read().strip())
+                f.close()
+            except (OSError, IOError) as e:
+                raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))

lib/ansible/parsing/quoting.py

@@ -0,0 +1,30 @@
+# (c) 2014 James Cammarata, <jcammarata@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def is_quoted(data):
+    return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\'
+
+def unquote(data):
+    ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
+    if is_quoted(data):
+        return data[1:-1]
+    return data

lib/ansible/parsing/splitter.py

@@ -23,6 +23,7 @@ import re
 import codecs
 
 from ansible.errors import AnsibleError
+from ansible.parsing.quoting import unquote
 
 # Decode escapes adapted from rspeer's answer here:
 # http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
@@ -263,12 +264,3 @@ def split_args(args):
         raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")
 
     return params
-
-def is_quoted(data):
-    return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\'
-
-def unquote(data):
-    ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
-    if is_quoted(data):
-        return data[1:-1]
-    return data

lib/ansible/parsing/yaml/constructor.py

@@ -19,9 +19,17 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from yaml.constructor import Constructor
+from yaml.constructor import Constructor, ConstructorError
+from yaml.nodes import MappingNode
 from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
 
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
 class AnsibleConstructor(Constructor):
     def __init__(self, file_name=None):
         self._ansible_file_name = file_name
@@ -35,10 +43,33 @@ class AnsibleConstructor(Constructor):
         data.ansible_pos = self._node_position_info(node)
 
     def construct_mapping(self, node, deep=False):
-        ret = AnsibleMapping(super(Constructor, self).construct_mapping(node, deep))
+        # This first part fixes a problem with pyyaml's handling of duplicate
-        ret.ansible_pos = self._node_position_info(node)
+        # dict keys.
+        # from SafeConstructor
+        if not isinstance(node, MappingNode):
+            raise ConstructorError(None, None,
+                    "expected a mapping node, but found %s" % node.id,
+                    node.start_mark)
+        self.flatten_mapping(node)
+        mapping = AnsibleMapping()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                hash(key)
+            except TypeError as exc:
+                raise ConstructorError("while constructing a mapping", node.start_mark,
+                        "found unacceptable key (%s)" % exc, key_node.start_mark)
 
-        return ret
+            if key in mapping:
+                display.warning('While constructing a mapping%s found a duplicate dict key (%s).  Using last value only.' % (node.start_mark, key_node.start_mark))
+
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+
+        # Add our extra information to the returned value
+        mapping.ansible_pos = self._node_position_info(node)
+
+        return mapping
 
     def construct_yaml_str(self, node):
         # Override the default string handling function

lib/ansible/playbook/__init__.py

@@ -21,9 +21,7 @@ __metaclass__ = type
 
 import os
 
-from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.errors import AnsibleParserError
-from ansible.parsing import DataLoader
-from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.play import Play
 from ansible.playbook.playbook_include import PlaybookInclude
 from ansible.plugins import get_all_plugin_loaders

lib/ansible/playbook/base.py

@@ -26,20 +26,16 @@ import uuid
 
 from functools import partial
 from inspect import getmembers
-from io import FileIO
 
 from ansible.compat.six import iteritems, string_types, text_type
 
 from jinja2.exceptions import UndefinedError
 
 from ansible.errors import AnsibleParserError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.playbook.attribute import Attribute, FieldAttribute
-from ansible.template import Templar
 from ansible.utils.boolean import boolean
-from ansible.utils.debug import debug
 from ansible.utils.vars import combine_vars, isidentifier
-from ansible.template import template
 
 class Base:
 

lib/ansible/vars/__init__.py

@@ -37,7 +37,6 @@ from ansible.cli import CLI
 from ansible.compat.six import string_types
 from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound
 from ansible.inventory.host import Host
-from ansible.parsing import DataLoader
 from ansible.plugins import lookup_loader
 from ansible.plugins.cache import FactCache
 from ansible.template import Templar

samples/multi_queues.py

@@ -13,7 +13,7 @@ from ansible.playbook.play_context import PlayContext
 from ansible.playbook.task import Task
 from ansible.executor.task_executor import TaskExecutor
 from ansible.executor.task_result import TaskResult
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.vars import VariableManager
 
 from ansible.utils.debug import debug

test/units/mock/loader.py

@@ -22,7 +22,7 @@ __metaclass__ = type
 import os
 
 from ansible.errors import AnsibleParserError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 
 class DictDataLoader(DataLoader):
 

test/units/parsing/test_data_loader.py

@@ -26,7 +26,7 @@ from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, mock_open
 from ansible.errors import AnsibleParserError
 
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.yaml.objects import AnsibleMapping
 
 class TestDataLoader(unittest.TestCase):

test/units/parsing/test_unquote.py

@@ -23,7 +23,7 @@ __metaclass__ = type
 from nose import tools
 from ansible.compat.tests import unittest
 
-from ansible.parsing.splitter import unquote
+from ansible.parsing.quoting import unquote
 
 
 # Tests using nose's test generators cannot use unittest base class.