When running all network tests at once, list the ones that failed at the end. (#27075)

* list failed tests for iosxr

* list failed tests for ovs

* list failed tests for junos

* list failed tests for ios

* list failed tests for eos

* list failed tests for nxos

* list failed tests for vyos

test/integration/eos.yaml

@@ -14,78 +14,100 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: eos_banner
         when: "limit_to in ['*', 'eos_banner']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'eos_banner' ]"
+            test_failed: true
     - block:
       - include_role:
           name: eos_command
         when: "limit_to in ['*', 'eos_command']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'eos_command' ]"
+            test_failed: true
     - block:
       - include_role:
           name: eos_config
         when: "limit_to in ['*', 'eos_config']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_config' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_template
         when: "limit_to in ['*', 'eos_template']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_template' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_facts
         when: "limit_to in ['*', 'eos_facts']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_facts' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_eapi
         when: "limit_to in ['*', 'eos_eapi']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_eapi' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_system
         when: "limit_to in ['*', 'eos_system']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_system' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_user
         when: "limit_to in ['*', 'eos_user']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_user' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_vlan
         when: "limit_to in ['*', 'eos_vlan']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_vlan' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: eos_vrf
         when: "limit_to in ['*', 'eos_vrf']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'eos_vrf' ]"
+            test_failed: true
 
 
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"

test/integration/ios.yaml

@@ -13,71 +13,92 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: ios_banner
         when: "limit_to in ['*', 'ios_banner']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'ios_banner' ]"
+            test_failed: true
     - block:
       - include_role:
           name: ios_command
         when: "limit_to in ['*', 'ios_command']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_command' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_config
         when: "limit_to in ['*', 'ios_config']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_config' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_facts
         when: "limit_to in ['*', 'ios_facts']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_facts' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_template
         when: "limit_to in ['*', 'ios_template']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_template' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_system
         when: "limit_to in ['*', 'ios_system']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_system' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_user
         when: "limit_to in ['*', 'ios_user']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_user' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_static_route
         when: "limit_to in ['*', 'ios_static_route']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_static_route' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: ios_logging
         when: "limit_to in ['*', 'ios_logging']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'ios_logging' ]"
+            test_failed: true
 
 
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"

test/integration/iosxr.yaml

@@ -14,63 +14,83 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: iosxr_command
         when: "limit_to in ['*', 'iosxr_command']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_command' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: iosxr_config
         when: "limit_to in ['*', 'iosxr_config']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_config' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: iosxr_facts
         when: "limit_to in ['*', 'iosxr_facts']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_facts' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: iosxr_template
         when: "limit_to in ['*', 'iosxr_template']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_template' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: iosxr_system
         when: "limit_to in ['*', 'iosxr_system']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            test_failed: true
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_system' ]"
 
     - block:
       - include_role:
           name: iosxr_user
         when: "limit_to in ['*', 'iosxr_user']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_user' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: iosxr_banner
         when: "limit_to in ['*', 'iosxr_banner']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_banner' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: iosxr_logging
         when: "limit_to in ['*', 'iosxr_logging']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'iosxr_logging' ]"
+            test_failed: true
 
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"

test/integration/junos.yaml

@@ -15,126 +15,160 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: junos_command
         when: "limit_to in ['*', 'junos_command']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_command' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_config
         when: "limit_to in ['*', 'junos_config']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_config' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_facts
         when: "limit_to in ['*', 'junos_facts']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_facts' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_netconf
         when: "limit_to in ['*', 'junos_netconf']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_netconf' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_rpc
         when: "limit_to in ['*', 'junos_rpc']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_rpc' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_template
         when: "limit_to in ['*', 'junos_template']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'junos_template' ]"
+            test_failed: true
     - block:
       - include_role:
           name: junos_vlan
         when: "limit_to in ['*', 'junos_vlan']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_vlan' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_interface
         when: "limit_to in ['*', 'junos_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'junos_interface' ]"
+            test_failed: true
     - block:
       - include_role:
           name: junos_banner
         when: "limit_to in ['*', 'junos_banner']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_banner' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_system
         when: "limit_to in ['*', 'junos_system']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'junos_system' ]"
+            test_failed: true
     - block:
       - include_role:
           name: junos_logging
         when: "limit_to in ['*', 'junos_logging']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_logging' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_user
         when: "limit_to in ['*', 'junos_user']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_user' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_static_route
         when: "limit_to in ['*', 'junos_static_route']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_static_route' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_linkagg
         when: "limit_to in ['*', 'junos_linkagg']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_linkagg' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_l3_interface
         when: "limit_to in ['*', 'junos_l3_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_l3_interface' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_lldp
         when: "limit_to in ['*', 'junos_lldp']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'junos_lldp' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: junos_lldp_interface
         when: "limit_to in ['*', 'junos_lldp_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
-
+            failed_modules: "{{ failed_modules }} + [ 'junos_lldp_interface' ]"
+            test_failed: true
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"

test/integration/nxos.yaml

@@ -14,105 +14,137 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: nxos_command
         when: "limit_to in ['*', 'nxos_command']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_command' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_config
         when: "limit_to in ['*', 'nxos_config']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_config' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_facts
         when: "limit_to in ['*', 'nxos_facts']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_facts' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_template
         when: "limit_to in ['*', 'nxos_template']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_template' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_nxapi
         when: "limit_to in ['*', 'nxos_nxapi']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_nxapi' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_evpn_global
         when: "limit_to in ['*', 'nxos_evpn_global']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_evpn_global' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_feature
         when: "limit_to in ['*', 'nxos_feature']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_feature' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_mtu
         when: "limit_to in ['*', 'nxos_mtu']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_mtu' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_system
         when: "limit_to in ['*', 'nxos_system']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_system' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_interface
         when: "limit_to in ['*', 'nxos_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_interface' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_user
         when: "limit_to in ['*', 'nxos_user']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_user' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_banner
         when: "limit_to in ['*', 'nxos_banner']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_banner' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_acl
         when: "limit_to in ['*', 'nxos_acl']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_acl' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: nxos_acl_interface
         when: "limit_to in ['*', 'nxos_acl_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'nxos_acl_interface' ]"
+            test_failed: true
 
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"

test/integration/ovs.yaml

@@ -15,16 +15,22 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: openvswitch_db
           _raw_params: openvswitch_db
         when: "limit_to in ['*', 'openvswitch_db']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'openvswitch_db' ]"
+            test_failed: true
 
 
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"

test/integration/vyos.yaml

@@ -14,84 +14,110 @@
   tasks:
     - set_fact:
         test_failed: false
+        failed_modules: []
     - block:
       - include_role:
           name: vyos_command
         when: "limit_to in ['*', 'vyos_command']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_command' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_config
         when: "limit_to in ['*', 'vyos_config']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_config' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_user
         when: "limit_to in ['*', 'vyos_user']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_user' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_banner
         when: "limit_to in ['*', 'vyos_banner']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_banner' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_linkagg
         when: "limit_to in ['*', 'vyos_linkagg']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_linkagg' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_static_route
         when: "limit_to in ['*', 'vyos_static_route']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_static_route' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_l3_interface
         when: "limit_to in ['*', 'vyos_l3_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_l3_interface' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_logging
         when: "limit_to in ['*', 'vyos_logging']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_logging' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_lldp
         when: "limit_to in ['*', 'vyos_lldp']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_lldp' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_lldp_interface
         when: "limit_to in ['*', 'vyos_lldp_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_lldp_interface' ]"
+            test_failed: true
 
     - block:
       - include_role:
           name: vyos_interface
         when: "limit_to in ['*', 'vyos_interface']"
       rescue:
-        - set_fact: test_failed=true
+        - set_fact:
+            failed_modules: "{{ failed_modules }} + [ 'vyos_interface' ]"
+            test_failed: true
 
 ###########
+    - debug: var=failed_modules
+      when: test_failed
+
     - name: Has any previous test failed?
       fail:
         msg: "One or more tests failed, check log for details"