diff --git a/test/integration/targets/order/order.yml b/test/integration/targets/order/order.yml
index 59691a2a76..62176b1e21 100644
--- a/test/integration/targets/order/order.yml
+++ b/test/integration/targets/order/order.yml
@@ -1,17 +1,17 @@
 - name: just plain order
   hosts: all
   gather_facts: false
-  order: '{{ myorder|default("inventory")}}'
+  order: '{{ myorder | default("inventory") }}'
   tasks:
-    - shell: echo '{{inventory_hostname}}' >> hostlist.txt
+    - shell: "echo '{{ inventory_hostname }}' >> hostlist.txt"
 
 - name: with serial
   hosts: all
   gather_facts: false
   serial: 1
-  order: '{{ myorder|default("inventory")}}'
+  order: '{{ myorder | default("inventory")}}'
   tasks:
-    - shell: echo '{{inventory_hostname}}' >> shostlist.txt
+    - shell: "echo '{{ inventory_hostname }}' >> shostlist.txt"
 
 - name: ensure everything works
   hosts: localhost
@@ -23,5 +23,17 @@
           - item.1 == shostlist[item.0]
       loop: '{{ lookup("indexed_items", inputlist) }}'
       vars:
-        hostlist: '{{lookup("file", "hostlist.txt").splitlines()}}'
-        shostlist: '{{lookup("file", "shostlist.txt").splitlines()}}'
+        hostlist: '{{ lookup("file", "hostlist.txt").splitlines() }}'
+        shostlist: '{{ lookup("file", "shostlist.txt").splitlines() }}'
+      when: myorder | default('inventory') != 'shuffle'
+
+    - name: Assert that shuffle worked
+      assert:
+        that:
+          - item.1 != hostlist[item.0] or item.1 in hostlist
+          - item.1 != hostlist[item.0] or item.1 in hostlist
+      loop: '{{ lookup("indexed_items", inputlist) }}'
+      vars:
+        hostlist: '{{ lookup("file", "hostlist.txt").splitlines() }}'
+        shostlist: '{{ lookup("file", "shostlist.txt").splitlines() }}'
+      when: myorder | default('inventory') == 'shuffle'
diff --git a/test/integration/targets/order/runme.sh b/test/integration/targets/order/runme.sh
index 16d86cca83..fdcdc33369 100755
--- a/test/integration/targets/order/runme.sh
+++ b/test/integration/targets/order/runme.sh
@@ -2,12 +2,23 @@
 
 set -eux
 
+cleanup () {
+    files="shostlist.txt hostlist.txt"
+    for file in $files; do
+        if [[ -f "$file" ]]; then
+            rm -f "$file"
+        fi
+    done
+}
+
 for EXTRA in '{"inputlist": ["hostB", "hostA", "hostD", "hostC"]}' \
              '{"myorder": "inventory", "inputlist": ["hostB", "hostA", "hostD", "hostC"]}' \
              '{"myorder": "sorted", "inputlist": ["hostA", "hostB", "hostC", "hostD"]}'  \
-		     '{"myorder": "reverse_sorted", "inputlist": ["hostD", "hostC", "hostB", "hostA"]}' \
-             '{"myorder": "reverse_inventory", "inputlist": ["hostC", "hostD", "hostA", "hostB"]}'
+             '{"myorder": "reverse_sorted", "inputlist": ["hostD", "hostC", "hostB", "hostA"]}' \
+             '{"myorder": "reverse_inventory", "inputlist": ["hostC", "hostD", "hostA", "hostB"]}' \
+             '{"myorder": "shuffle", "inputlist": ["hostC", "hostD", "hostA", "hostB"]}'
 do
-	rm shostlist.txt hostlist.txt || true
-	ansible-playbook order.yml --forks 1 -i inventory -e "$EXTRA" "$@"
+    cleanup
+    ansible-playbook order.yml --forks 1 -i inventory -e "$EXTRA" "$@" || cleanup
 done
+cleanup