Adding more fixes for integration testing under v2

v2/ansible/executor/task_result.py

@@ -43,6 +43,9 @@ class TaskResult:
         return self._check_key('skipped')
 
     def is_failed(self):
+        if 'failed_when_result' in self._result:
+            return self._check_key('failed_when_result')
+        else:
             return self._check_key('failed') or self._result.get('rc', 0) != 0
 
     def is_unreachable(self):

v2/ansible/module_utils/database.py

@@ -0,0 +1,128 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+class SQLParseError(Exception):
+    pass
+
+class UnclosedQuoteError(SQLParseError):
+    pass
+
+# maps a type of identifier to the maximum number of dot levels that are
+# allowed to specifiy that identifier.  For example, a database column can be
+# specified by up to 4 levels: database.schema.table.column
+_PG_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, schema=2, table=3, column=4, role=1)
+_MYSQL_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, table=2, column=3, role=1, vars=1)
+
+def _find_end_quote(identifier, quote_char):
+    accumulate = 0
+    while True:
+        try:
+            quote = identifier.index(quote_char)
+        except ValueError:
+            raise UnclosedQuoteError
+        accumulate = accumulate + quote
+        try:
+            next_char = identifier[quote+1]
+        except IndexError:
+            return accumulate
+        if next_char == quote_char:
+            try:
+                identifier = identifier[quote+2:]
+                accumulate = accumulate + 2
+            except IndexError:
+                raise UnclosedQuoteError
+        else:
+            return accumulate
+
+
+def _identifier_parse(identifier, quote_char):
+    if not identifier:
+        raise SQLParseError('Identifier name unspecified or unquoted trailing dot')
+
+    already_quoted = False
+    if identifier.startswith(quote_char):
+        already_quoted = True
+        try:
+            end_quote = _find_end_quote(identifier[1:], quote_char=quote_char) + 1
+        except UnclosedQuoteError:
+            already_quoted = False
+        else:
+            if end_quote < len(identifier) - 1:
+                if identifier[end_quote+1] == '.':
+                    dot = end_quote + 1
+                    first_identifier = identifier[:dot]
+                    next_identifier = identifier[dot+1:]
+                    further_identifiers = _identifier_parse(next_identifier, quote_char)
+                    further_identifiers.insert(0, first_identifier)
+                else:
+                    raise SQLParseError('User escaped identifiers must escape extra quotes')
+            else:
+                further_identifiers = [identifier]
+
+    if not already_quoted:
+        try:
+            dot = identifier.index('.')
+        except ValueError:
+            identifier = identifier.replace(quote_char, quote_char*2)
+            identifier = ''.join((quote_char, identifier, quote_char))
+            further_identifiers = [identifier]
+        else:
+            if dot == 0 or dot >= len(identifier) - 1:
+                identifier = identifier.replace(quote_char, quote_char*2)
+                identifier = ''.join((quote_char, identifier, quote_char))
+                further_identifiers = [identifier]
+            else:
+                first_identifier = identifier[:dot]
+                next_identifier = identifier[dot+1:]
+                further_identifiers = _identifier_parse(next_identifier, quote_char)
+                first_identifier = first_identifier.replace(quote_char, quote_char*2)
+                first_identifier = ''.join((quote_char, first_identifier, quote_char))
+                further_identifiers.insert(0, first_identifier)
+
+    return further_identifiers
+
+
+def pg_quote_identifier(identifier, id_type):
+    identifier_fragments = _identifier_parse(identifier, quote_char='"')
+    if len(identifier_fragments) > _PG_IDENTIFIER_TO_DOT_LEVEL[id_type]:
+        raise SQLParseError('PostgreSQL does not support %s with more than %i dots' % (id_type, _PG_IDENTIFIER_TO_DOT_LEVEL[id_type]))
+    return '.'.join(identifier_fragments)
+
+def mysql_quote_identifier(identifier, id_type):
+    identifier_fragments = _identifier_parse(identifier, quote_char='`')
+    if len(identifier_fragments) > _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type]:
+        raise SQLParseError('MySQL does not support %s with more than %i dots' % (id_type, _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type]))
+
+    special_cased_fragments = []
+    for fragment in identifier_fragments:
+        if fragment == '`*`':
+            special_cased_fragments.append('*')
+        else:
+            special_cased_fragments.append(fragment)
+
+    return '.'.join(special_cased_fragments)

v2/ansible/parsing/mod_args.py

@@ -267,6 +267,10 @@ class ModuleArgsParser:
         # if we didn't see any module in the task at all, it's not a task really
         if action is None:
             raise AnsibleParserError("no action detected in task", obj=self._task_ds)
+        # FIXME: disabled for now, as there are other places besides the shell/script modules where
+        #        having variables as the sole param for the module is valid (include_vars, add_host, and group_by?)
+        #elif args.get('_raw_params', '') != '' and action not in ('command', 'shell', 'script', 'include_vars'):
+        #    raise AnsibleParserError("this task has extra params, which is only allowed in the command, shell or script module.", obj=self._task_ds)
 
         # shell modules require special handling
         (action, args) = self._handle_shell_weirdness(action, args)

v2/ansible/playbook/block.py

@@ -45,10 +45,20 @@ class Block(Base, Conditional, Taggable):
 
         super(Block, self).__init__()
 
-    def get_variables(self):
-        # blocks do not (currently) store any variables directly,
-        # so we just return an empty dict here
-        return dict()
+    def get_vars(self):
+        '''
+        Blocks do not store variables directly, however they may be a member
+        of a role or task include which does, so return those if present.
+        '''
+
+        all_vars = dict()
+
+        if self._role:
+            all_vars.update(self._role.get_vars())
+        if self._task_include:
+            all_vars.update(self._task_include.get_vars())
+
+        return all_vars
 
     @staticmethod
     def load(data, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
@@ -73,17 +83,49 @@ class Block(Base, Conditional, Taggable):
         return ds
 
     def _load_block(self, attr, ds):
-        return load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers)
+        return load_list_of_tasks(
+            ds,
+            block=self,
+            role=self._role,
+            task_include=self._task_include,
+            variable_manager=self._variable_manager,
+            loader=self._loader,
+            use_handlers=self._use_handlers,
+        )
 
     def _load_rescue(self, attr, ds):
-        return load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers)
+        return load_list_of_tasks(
+            ds,
+            block=self,
+            role=self._role,
+            task_include=self._task_include,
+            variable_manager=self._variable_manager,
+            loader=self._loader,
+            use_handlers=self._use_handlers,
+        )
 
     def _load_always(self, attr, ds):
-        return load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers)
+        return load_list_of_tasks(
+            ds, 
+            block=self, 
+            role=self._role, 
+            task_include=self._task_include,
+            variable_manager=self._variable_manager, 
+            loader=self._loader, 
+            use_handlers=self._use_handlers,
+        )
 
     # not currently used
     #def _load_otherwise(self, attr, ds):
-    #    return self._load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers)
+    #    return load_list_of_tasks(
+    #        ds, 
+    #        block=self, 
+    #        role=self._role, 
+    #        task_include=self._task_include,
+    #        variable_manager=self._variable_manager, 
+    #        loader=self._loader, 
+    #        use_handlers=self._use_handlers,
+    #    )
 
     def compile(self):
         '''
@@ -125,6 +167,8 @@ class Block(Base, Conditional, Taggable):
 
         if self._role is not None:
             data['role'] = self._role.serialize()
+        if self._task_include is not None:
+            data['task_include'] = self._task_include.serialize()
 
         return data
 
@@ -134,6 +178,8 @@ class Block(Base, Conditional, Taggable):
         serialize method
         '''
 
+        from ansible.playbook.task_include import TaskInclude
+
         # unpack the when attribute, which is the only one we want
         self.when = data.get('when')
 
@@ -144,7 +190,17 @@ class Block(Base, Conditional, Taggable):
             r.deserialize(role_data)
             self._role = r
 
+        # if there was a serialized task include, unpack it too
+        ti_data = data.get('task_include')
+        if ti_data:
+            ti = TaskInclude()
+            ti.deserialize(ti_data)
+            self._task_include = ti
+
     def evaluate_conditional(self, all_vars):
+        if self._task_include is not None:
+            if not self._task_include.evaluate_conditional(all_vars):
+                return False
         if self._parent_block is not None:
             if not self._parent_block.evaluate_conditional(all_vars):
                 return False
@@ -168,3 +224,6 @@ class Block(Base, Conditional, Taggable):
         elif self._role:
             self._role.set_loader(loader)
 
+        if self._task_include:
+            self._task_include.set_loader(loader)
+

v2/ansible/playbook/role/definition.py

@@ -44,8 +44,8 @@ class RoleDefinition(Base, Conditional, Taggable):
         self._role_params  = dict()
         super(RoleDefinition, self).__init__()
 
-    def __repr__(self):
-        return 'ROLEDEF: ' + self._attributes.get('role', '<no name set>')
+    #def __repr__(self):
+    #    return 'ROLEDEF: ' + self._attributes.get('role', '<no name set>')
 
     @staticmethod
     def load(data, variable_manager=None, loader=None):

v2/ansible/playbook/task.py

@@ -33,6 +33,7 @@ from ansible.playbook.block import Block
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.role import Role
 from ansible.playbook.taggable import Taggable
+from ansible.playbook.task_include import TaskInclude
 
 class Task(Base, Conditional, Taggable):
 
@@ -189,16 +190,23 @@ class Task(Base, Conditional, Taggable):
     def post_validate(self, all_vars=dict(), fail_on_undefined=True):
         '''
         Override of base class post_validate, to also do final validation on
-        the block to which this task belongs.
+        the block and task include (if any) to which this task belongs.
         '''
 
         if self._block:
             self._block.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined)
+        if self._task_include:
+            self._task_include.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined)
 
         super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined)
 
     def get_vars(self):
-        all_vars = self.serialize()
+        all_vars = dict()
+        if self._task_include:
+            all_vars.update(self._task_include.get_vars())
+
+        all_vars.update(self.serialize())
+
         if 'tags' in all_vars:
             del all_vars['tags']
         if 'when' in all_vars:
@@ -242,6 +250,9 @@ class Task(Base, Conditional, Taggable):
         if self._role:
             data['role'] = self._role.serialize()
 
+        if self._task_include:
+            data['task_include'] = self._task_include.serialize()
+
         return data
 
     def deserialize(self, data):
@@ -261,6 +272,13 @@ class Task(Base, Conditional, Taggable):
             self._role = r
             del data['role']
 
+        ti_data = data.get('task_include')
+        if ti_data:
+            ti = TaskInclude()
+            ti.deserialize(ti_data)
+            self._task_include = ti
+            del data['task_include']
+
         super(Task, self).deserialize(data)
 
     def evaluate_conditional(self, all_vars):
@@ -271,6 +289,9 @@ class Task(Base, Conditional, Taggable):
         if self._block is not None:
             if not self._block.evaluate_conditional(all_vars):
                 return False
+        if self._task_include is not None:
+            if not self._task_include.evaluate_conditional(all_vars):
+                return False
         return super(Task, self).evaluate_conditional(all_vars)
 
     def evaluate_tags(self, only_tags, skip_tags, all_vars):
@@ -293,6 +314,8 @@ class Task(Base, Conditional, Taggable):
 
         if self._block:
             self._block.set_loader(loader)
+        if self._task_include:
+            self._task_include.set_loader(loader)
 
         for dep in self._dep_chain:
             dep.set_loader(loader)

v2/ansible/playbook/task_include.py

@@ -24,14 +24,16 @@ from ansible.parsing.splitter import split_args, parse_kv
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
 from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.base import Base
+from ansible.playbook.conditional import Conditional
 from ansible.playbook.helpers import load_list_of_blocks, compile_block_list
+from ansible.playbook.taggable import Taggable
 from ansible.plugins import lookup_loader
 
 
 __all__ = ['TaskInclude']
 
 
-class TaskInclude(Base):
+class TaskInclude(Base, Conditional, Taggable):
 
     '''
     A class used to wrap the use of `include: /some/other/file.yml`
@@ -164,3 +166,77 @@ class TaskInclude(Base):
         task_list.extend(compile_block_list(self._task_blocks))
         return task_list
 
+    def get_vars(self):
+        '''
+        Returns the vars for this task include, but also first merges in
+        those from any parent task include which may exist.
+        '''
+
+        all_vars = dict()
+        if self._task_include:
+            all_vars.update(self._task_include.get_vars())
+        all_vars.update(self.vars)
+        return all_vars
+
+    def serialize(self):
+
+        data = super(TaskInclude, self).serialize()
+
+        if self._block:
+            data['block'] = self._block.serialize()
+
+        if self._role:
+            data['role'] = self._role.serialize()
+
+        if self._task_include:
+            data['task_include'] = self._task_include.serialize()
+
+        return data
+
+    def deserialize(self, data):
+
+        # import here to prevent circular importing issues
+        from ansible.playbook.block import Block
+        from ansible.playbook.role import Role
+
+        block_data = data.get('block')
+        if block_data:
+            b = Block()
+            b.deserialize(block_data)
+            self._block = b
+            del data['block']
+
+        role_data = data.get('role')
+        if role_data:
+            r = Role()
+            r.deserialize(role_data)
+            self._role = r
+            del data['role']
+
+        ti_data = data.get('task_include')
+        if ti_data:
+            ti = TaskInclude()
+            ti.deserialize(ti_data)
+            self._task_include = ti
+            del data['task_include']
+
+        super(TaskInclude, self).deserialize(data)
+
+    def evaluate_conditional(self, all_vars):
+        if self._task_include is not None:
+            if not self._task_include.evaluate_conditional(all_vars):
+                return False
+        if self._block is not None:
+            if not self._block.evaluate_conditional(all_vars):
+                return False
+        elif self._role is not None:
+            if not self._role.evaluate_conditional(all_vars):
+                return False
+        return super(TaskInclude, self).evaluate_conditional(all_vars)
+
+    def set_loader(self, loader):
+        self._loader = loader
+        if self._block:
+            self._block.set_loader(loader)
+        elif self._task_include:
+            self._task_include.set_loader(loader)

v2/ansible/plugins/lookup/first_found.py

@@ -121,6 +121,7 @@ import os
 
 from ansible.plugins.lookup import LookupBase
 from ansible.template import Templar
+from ansible.utils.boolean import boolean
 
 class LookupModule(LookupBase):
 

v2/ansible/plugins/strategies/linear.py

@@ -70,7 +70,7 @@ class StrategyModule(StrategyBase):
                                 debug("'%s' skipped because role has already run" % task)
                                 continue
 
-                        if not task.evaluate_tags(connection_info.only_tags, connection_info.skip_tags, task_vars):
+                        if not task.evaluate_tags(connection_info.only_tags, connection_info.skip_tags, task_vars) and task.action != 'setup':
                             debug("'%s' failed tag evaluation" % task)
                             continue