diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index a74a93900a..35e6a96125 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -17,6 +17,13 @@ ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc
 
 retry.py pip install tox --disable-pip-version-check
 
-ansible-test compile --color -v
-ansible-test sanity --color -v --junit --tox --skip-test ansible-doc --python 2.7
-ansible-test sanity --color -v --junit --tox --test ansible-doc --coverage
+errors=0
+
+ansible-test compile --color -v || ((errors++))
+ansible-test sanity --color -v --junit --tox --skip-test ansible-doc --python 2.7 || ((errors++))
+ansible-test sanity --color -v --junit --tox --test ansible-doc --coverage || ((errors++))
+
+if [ ${errors} -gt 0 ]; then
+    echo "${errors} of the above ansible-test command(s) failed."
+    exit 1
+fi