Add ansible-test options and docs. (#26823)

* All integration commands support --continue-on-error
* The network-integration command supports --inventory
* Add landing page for compile test docs.
* Add bot documentation links.

docs/docsite/rst/dev_guide/testing/compile/index.rst

@@ -0,0 +1,4 @@
+Compile Tests
+=============
+
+See :doc:`testing_compile` for more information.

test/runner/lib/config.py

@@ -140,6 +140,7 @@ class IntegrationConfig(TestConfig):
         self.start_at_task = args.start_at_task  # type: str
         self.allow_destructive = args.allow_destructive if 'allow_destructive' in args else False  # type: bool
         self.retry_on_error = args.retry_on_error  # type: bool
+        self.continue_on_error = args.continue_on_error  # type: bool
         self.debug_strategy = args.debug_strategy  # type: bool
         self.changed_all_target = args.changed_all_target  # type: str
         self.tags = args.tags
@@ -182,6 +183,7 @@ class NetworkIntegrationConfig(IntegrationConfig):
         super(NetworkIntegrationConfig, self).__init__(args, 'network-integration')
 
         self.platform = args.platform  # type: list [str]
+        self.inventory = args.inventory  # type: str
 
 
 class UnitsConfig(TestConfig):

test/runner/lib/executor.py

@@ -252,10 +252,23 @@ def command_network_integration(args):
     """
     :type args: NetworkIntegrationConfig
     """
-    filename = 'test/integration/inventory.networking'
+    default_filename = 'test/integration/inventory.networking'
 
-    if not args.explain and not args.platform and not os.path.isfile(filename):
+    if args.inventory:
-        raise ApplicationError('Use the --platform option or provide an inventory file (see %s.template).' % filename)
+        filename = os.path.join('test/integration', args.inventory)
+    else:
+        filename = default_filename
+
+    if not args.explain and not args.platform and not os.path.exists(filename):
+        if args.inventory:
+            filename = os.path.abspath(filename)
+
+        raise ApplicationError(
+            'Inventory not found: %s\n'
+            'Use --inventory to specify the inventory path.\n'
+            'Use --platform to provision resources and generate an inventory file.\n'
+            'See also inventory template: %s.template' % (filename, default_filename)
+        )
 
     internal_targets = command_integration_filter(args, walk_network_integration_targets())
     platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
@@ -500,6 +513,8 @@ def command_integration_filtered(args, targets):
     :type targets: tuple[IntegrationTarget]
     """
     found = False
+    passed = []
+    failed = []
 
     targets_iter = iter(targets)
 
@@ -568,7 +583,14 @@ def command_integration_filtered(args, targets):
                     display.verbosity = args.verbosity = 6
 
             original_environment.validate(target.name, throw=True)
-        except:
+            passed.append(target)
+        except Exception as ex:
+            failed.append(target)
+
+            if args.continue_on_error:
+                display.error(ex)
+                continue
+
             display.notice('To resume at this test target, use the option: --start-at %s' % target.name)
 
             next_target = next(targets_iter, None)
@@ -580,6 +602,10 @@ def command_integration_filtered(args, targets):
         finally:
             display.verbosity = args.verbosity = verbosity
 
+    if failed:
+        raise ApplicationError('The %d integration test(s) listed below (out of %d) failed. See error output above for details:\n%s' % (
+            len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed)))
+
 
 def integration_environment(args, target, cmd):
     """
@@ -642,7 +668,7 @@ def command_integration_role(args, target, start_at_task):
         hosts = 'windows'
         gather_facts = False
     elif isinstance(args, NetworkIntegrationConfig):
-        inventory = 'inventory.networking'
+        inventory = args.inventory or 'inventory.networking'
         hosts = target.name[:target.name.find('_')]
         gather_facts = False
         if hosts == 'net':

test/runner/lib/test.py

@@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function
 
 import datetime
 import json
+import os
 
 from lib.util import (
     display,
@@ -260,6 +261,7 @@ class TestFailure(TestResult):
         """
         message = self.format_title()
         output = self.format_block()
+        docs = self.find_docs()
 
         if self.messages:
             verified = all((m.confidence or 0) >= 50 for m in self.messages)
@@ -268,6 +270,7 @@ class TestFailure(TestResult):
 
         bot_data = dict(
             verified=verified,
+            docs=docs,
             results=[
                 dict(
                     message=message,
@@ -307,6 +310,25 @@ class TestFailure(TestResult):
 
         return command
 
+    def find_docs(self):
+        """
+        :rtype: str
+        """
+        testing_docs_url = 'https://docs.ansible.com/ansible/devel/dev_guide/testing'
+        testing_docs_dir = 'docs/docsite/rst/dev_guide/testing'
+
+        url = '%s/%s/' % (testing_docs_url, self.command)
+        path = os.path.join(testing_docs_dir, self.command)
+
+        if self.test:
+            url += '%s.html' % self.test
+            path = os.path.join(path, '%s.rst' % self.test)
+
+        if os.path.exists(path):
+            return url
+
+        return None
+
     def format_title(self):
         """
         :rtype: str

test/runner/test.py

@@ -216,6 +216,10 @@ def parse_args():
                              action='store_true',
                              help='retry failed test with increased verbosity')
 
+    integration.add_argument('--continue-on-error',
+                             action='store_true',
+                             help='continue after failed test')
+
     integration.add_argument('--debug-strategy',
                              action='store_true',
                              help='run test playbooks using the debug strategy')
@@ -251,6 +255,10 @@ def parse_args():
                                      action='append',
                                      help='network platform/version').completer = complete_network_platform
 
+    network_integration.add_argument('--inventory',
+                                     metavar='PATH',
+                                     help='path to inventory used for tests')
+
     windows_integration = subparsers.add_parser('windows-integration',
                                                 parents=[integration],
                                                 help='windows integration tests')