From 50f2313b47eed5eb338dc28afcb0ef9cdb34e339 Mon Sep 17 00:00:00 2001
From: John R Barker <john@johnrbarker.com>
Date: Tue, 4 Jul 2017 18:18:02 +0100
Subject: [PATCH] Continue running network tests after failure (#26407)

Previously tests would stop running on the first failure.
With this PR we move onto running the tests for the next module. This
allows DCI runs to have a fuller picture of what's broken.

This only effects manual and DCI runs, it doesn't effect Shippable runs
as this the modules are tested directly.
---
 test/integration/asa.yaml               |  37 +++++++-
 test/integration/eos.yaml               |  94 ++++++++++++++++---
 test/integration/ios.yaml               |  77 ++++++++++++++--
 test/integration/iosxr.yaml             |  69 ++++++++++++--
 test/integration/junos.yaml             | 102 +++++++++++++++++---
 test/integration/nxos.yaml              | 118 +++++++++++++++++++++---
 test/integration/ops.yaml               |  22 ++++-
 test/integration/ovs.yaml               |  22 ++++-
 test/integration/platform_agnostic.yaml |  86 +++++++++++++++--
 test/integration/vyos.yaml              |  46 ++++++++-
 10 files changed, 596 insertions(+), 77 deletions(-)

diff --git a/test/integration/asa.yaml b/test/integration/asa.yaml
index 62fef0d4c7..a21bf7588a 100644
--- a/test/integration/asa.yaml
+++ b/test/integration/asa.yaml
@@ -7,7 +7,36 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: asa_command, when: "limit_to in ['*', 'asa_command']" }
-    - { role: asa_config, when: "limit_to in ['*', 'asa_config']" }
-    - { role: asa_acl, when: "limit_to in ['*', 'asa_acl']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: asa_command
+        when: "limit_to in ['*', 'asa_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: asa_config
+        when: "limit_to in ['*', 'asa_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: asa_acl
+        when: "limit_to in ['*', 'asa_acl']"
+      rescue:
+        - set_fact: test_failed=true
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/eos.yaml b/test/integration/eos.yaml
index 9e2e4c3716..59aed8a99f 100644
--- a/test/integration/eos.yaml
+++ b/test/integration/eos.yaml
@@ -7,14 +7,86 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: eos_banner, when: "limit_to in ['*', 'eos_banner']" }
-    - { role: eos_command, when: "limit_to in ['*', 'eos_command']" }
-    - { role: eos_config, when: "limit_to in ['*', 'eos_config']" }
-    - { role: eos_template, when: "limit_to in ['*', 'eos_template']" }
-    - { role: eos_facts, when: "limit_to in ['*', 'eos_facts']" }
-    - { role: eos_eapi, debug: yes, when: "limit_to in ['*', 'eos_eapi']" }
-    - { role: eos_system, debug: yes, when: "limit_to in ['*', 'eos_system']" }
-    - { role: eos_user, when: "limit_to in ['*', 'eos_user']" }
-    - { role: eos_vlan, debug: yes, when: "limit_to in ['*', 'eos_vlan']" }
-    - { role: eos_vrf, debug: yes, when: "limit_to in ['*', 'eos_vrf']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: eos_banner
+        when: "limit_to in ['*', 'eos_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_command
+        when: "limit_to in ['*', 'eos_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_config
+        when: "limit_to in ['*', 'eos_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_template
+        when: "limit_to in ['*', 'eos_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_facts
+        when: "limit_to in ['*', 'eos_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_eapi
+        when: "limit_to in ['*', 'eos_eapi']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_system
+        when: "limit_to in ['*', 'eos_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_user
+        when: "limit_to in ['*', 'eos_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_vlan
+        when: "limit_to in ['*', 'eos_vlan']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_vrf
+        when: "limit_to in ['*', 'eos_vrf']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/ios.yaml b/test/integration/ios.yaml
index 720a4fe136..a47b56f82e 100644
--- a/test/integration/ios.yaml
+++ b/test/integration/ios.yaml
@@ -6,13 +6,72 @@
   vars:
     limit_to: "*"
     debug: false
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
 
-  roles:
-    - { role: ios_banner, when: "limit_to in ['*', 'ios_banner']" }
-    - { role: ios_command, when: "limit_to in ['*', 'ios_command']" }
-    - { role: ios_config, when: "limit_to in ['*', 'ios_config']" }
-    - { role: ios_facts, when: "limit_to in ['*', 'ios_facts']" }
-    - { role: ios_template, when: "limit_to in ['*', 'ios_template']" }
-    - { role: ios_system, when: "limit_to in ['*', 'ios_system']" }
-    - { role: ios_user, when: "limit_to in ['*', 'ios_user']" }
-    - { role: ios_static_route, when: "limit_to in ['*', 'ios_static_route']" }
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: ios_banner
+        when: "limit_to in ['*', 'ios_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_command
+        when: "limit_to in ['*', 'ios_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_config
+        when: "limit_to in ['*', 'ios_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_facts
+        when: "limit_to in ['*', 'ios_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_template
+        when: "limit_to in ['*', 'ios_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_system
+        when: "limit_to in ['*', 'ios_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_user
+        when: "limit_to in ['*', 'ios_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_static_route
+        when: "limit_to in ['*', 'ios_static_route']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/iosxr.yaml b/test/integration/iosxr.yaml
index 8b9f968b22..00aa67698a 100644
--- a/test/integration/iosxr.yaml
+++ b/test/integration/iosxr.yaml
@@ -7,11 +7,64 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: iosxr_command, when: "limit_to in ['*', 'iosxr_command']" }
-    - { role: iosxr_config, when: "limit_to in ['*', 'iosxr_config']" }
-    - { role: iosxr_facts, when: "limit_to in ['*', 'iosxr_facts']" }
-    - { role: iosxr_template, when: "limit_to in ['*', 'iosxr_template']" }
-    - { role: iosxr_system, when: "limit_to in ['*', 'iosxr_system']" }
-    - { role: iosxr_user, when: "limit_to in ['*', 'iosxr_user']" }
-    - { role: iosxr_banner, when: "limit_to in ['*', 'iosxr_banner']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: iosxr_command
+        when: "limit_to in ['*', 'iosxr_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_config
+        when: "limit_to in ['*', 'iosxr_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_facts
+        when: "limit_to in ['*', 'iosxr_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_template
+        when: "limit_to in ['*', 'iosxr_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_system
+        when: "limit_to in ['*', 'iosxr_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_user
+        when: "limit_to in ['*', 'iosxr_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_banner
+        when: "limit_to in ['*', 'iosxr_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/junos.yaml b/test/integration/junos.yaml
index cd3fd7de6d..cc825def43 100644
--- a/test/integration/junos.yaml
+++ b/test/integration/junos.yaml
@@ -7,15 +7,93 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: junos_command, when: "limit_to in ['*', 'junos_command']" }
-    - { role: junos_config, when: "limit_to in ['*', 'junos_config']" }
-    - { role: junos_facts, when: "limit_to in ['*', 'junos_facts']" }
-    - { role: junos_netconf, when: "limit_to in ['*', 'junos_netconf']" }
-    - { role: junos_rpc, when: "limit_to in ['*', 'junos_rpc']" }
-    - { role: junos_template, when: "limit_to in ['*', 'junos_template']" }
-    - { role: junos_vlan, when: "limit_to in ['*', 'junos_vlan']" }
-    - { role: junos_interface, when: "limit_to in ['*', 'junos_interface']" }
-    - { role: junos_banner, when: "limit_to in ['*', 'junos_banner']" }
-    - { role: junos_system, when: "limit_to in ['*', 'junos_system']" }
-    - { role: junos_logging, when: "limit_to in ['*', 'junos_logging']" }
+
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: junos_command
+        when: "limit_to in ['*', 'junos_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_config
+        when: "limit_to in ['*', 'junos_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_facts
+        when: "limit_to in ['*', 'junos_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_netconf
+        when: "limit_to in ['*', 'junos_netconf']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_rpc
+        when: "limit_to in ['*', 'junos_rpc']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_template
+        when: "limit_to in ['*', 'junos_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_vlan
+        when: "limit_to in ['*', 'junos_vlan']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_interface
+        when: "limit_to in ['*', 'junos_interface']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_banner
+        when: "limit_to in ['*', 'junos_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_system
+        when: "limit_to in ['*', 'junos_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: junos_logging
+        when: "limit_to in ['*', 'junos_logging']"
+      rescue:
+        - set_fact: test_failed=true
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/nxos.yaml b/test/integration/nxos.yaml
index 3005389662..e15004f422 100644
--- a/test/integration/nxos.yaml
+++ b/test/integration/nxos.yaml
@@ -7,17 +7,107 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: nxos_command, when: "limit_to in ['*', 'nxos_command']" }
-    - { role: nxos_config, when: "limit_to in ['*', 'nxos_config']" }
-    - { role: nxos_facts, when: "limit_to in ['*', 'nxos_facts']" }
-    - { role: nxos_template, when: "limit_to in ['*', 'nxos_template']" }
-    - { role: nxos_nxapi, when: "limit_to in ['*', 'nxos_nxapi']" }
-    - { role: nxos_evpn_global, when: "limit_to in ['*', 'nxos_evpn_global']" }
-    - { role: nxos_feature, when: "limit_to in ['*', 'nxos_feature']" }
-    - { role: nxos_mtu, when: "limit_to in ['*', 'nxos_mtu']" }
-    - { role: nxos_system, when: "limit_to in ['*', 'nxos_system']" }
-    - { role: nxos_interface, when: "limit_to in ['*', 'nxos_interface']" }
-    - { role: nxos_user, when: "limit_to in ['*', 'nxos_user']" }
-    - { role: nxos_banner, when: "limit_to in ['*', 'nxos_banner']" }
-    - { role: nxos_acl, when: "limit_to in ['*', 'nxos_acl']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: nxos_command
+        when: "limit_to in ['*', 'nxos_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_config
+        when: "limit_to in ['*', 'nxos_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_facts
+        when: "limit_to in ['*', 'nxos_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_template
+        when: "limit_to in ['*', 'nxos_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_nxapi
+        when: "limit_to in ['*', 'nxos_nxapi']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_evpn_global
+        when: "limit_to in ['*', 'nxos_evpn_global']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_feature
+        when: "limit_to in ['*', 'nxos_feature']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_mtu
+        when: "limit_to in ['*', 'nxos_mtu']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_system
+        when: "limit_to in ['*', 'nxos_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_interface
+        when: "limit_to in ['*', 'nxos_interface']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_user
+        when: "limit_to in ['*', 'nxos_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_banner
+        when: "limit_to in ['*', 'nxos_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: nxos_acl
+        when: "limit_to in ['*', 'nxos_acl']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/ops.yaml b/test/integration/ops.yaml
index f371bc0c98..782d0a0d5b 100644
--- a/test/integration/ops.yaml
+++ b/test/integration/ops.yaml
@@ -7,5 +7,23 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: ops_command, when: "limit_to in ['*', 'ops_command']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: ops_command
+        when: "limit_to in ['*', 'ops_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/ovs.yaml b/test/integration/ovs.yaml
index f3f812673a..1fb4ef9a44 100644
--- a/test/integration/ovs.yaml
+++ b/test/integration/ovs.yaml
@@ -8,5 +8,23 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: openvswitch_db, when: "limit_to in ['*', 'openvswitch_db']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: openvswitch_db
+        when: "limit_to in ['*', 'openvswitch_db']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/platform_agnostic.yaml b/test/integration/platform_agnostic.yaml
index c4ec0bdc18..614567e581 100644
--- a/test/integration/platform_agnostic.yaml
+++ b/test/integration/platform_agnostic.yaml
@@ -7,13 +7,79 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: net_system, when: "limit_to in ['*', 'net_system']" }
-    - { role: net_banner, when: "limit_to in ['*', 'net_banner']" }
-    - { role: net_command, when: "limit_to in ['*', 'net_command']" }
-    - { role: net_user, when: "limit_to in ['*', 'net_user']" }
-    - { role: net_vlan,  when: "limit_to in ['*', 'net_vlan']" }
-    - { role: net_vrf, when: "limit_to in ['*', 'net_vrf']" }
-    - { role: net_interface, when: "limit_to in ['*', 'net_interface']" }
-    - { role: net_static_route, when: "limit_to in ['*', 'net_static_route']" }
-    - { role: net_logging, when: "limit_to in ['*', 'net_logging']" }
+
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: net_system
+        when: "limit_to in ['*', 'net_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_banner
+        when: "limit_to in ['*', 'net_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_command
+        when: "limit_to in ['*', 'net_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_user
+        when: "limit_to in ['*', 'net_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_vlan
+        when: "limit_to in ['*', 'net_vlan']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_vrf
+        when: "limit_to in ['*', 'net_vrf']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_interface
+        when: "limit_to in ['*', 'net_interface']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_static_route
+        when: "limit_to in ['*', 'net_static_route']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: net_logging
+        when: "limit_to in ['*', 'net_logging']"
+      rescue:
+        - set_fact: test_failed=true
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file
diff --git a/test/integration/vyos.yaml b/test/integration/vyos.yaml
index b80ab151a7..5630d79e6d 100644
--- a/test/integration/vyos.yaml
+++ b/test/integration/vyos.yaml
@@ -7,8 +7,44 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: vyos_command, when: "limit_to in ['*', 'vyos_command']" }
-    - { role: vyos_config, when: "limit_to in ['*', 'vyos_config']" }
-    - { role: vyos_user, when: "limit_to in ['*', 'vyos_user']" }
-    - { role: vyos_banner, when: "limit_to in ['*', 'vyos_banner']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: vyos_command
+        when: "limit_to in ['*', 'vyos_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: vyos_config
+        when: "limit_to in ['*', 'vyos_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: vyos_user
+        when: "limit_to in ['*', 'vyos_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: vyos_banner
+        when: "limit_to in ['*', 'vyos_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
\ No newline at end of file