Improve on-demand coverage for Shippable.

test/runner/lib/executor.py

@@ -965,7 +965,7 @@ def detect_changes(args):
     :type args: TestConfig
     :rtype: list[str] | None
     """
-    if is_shippable():
+    if args.changed and is_shippable():
         display.info('Shippable detected, collecting parameters from environment.')
         paths = detect_changes_shippable(args)
     elif args.changed_from or args.changed_path:

test/utils/shippable/cloud.sh

@@ -9,4 +9,4 @@ image="ansible/ansible:${args[1]}"
 target="posix/ci/cloud/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \

test/utils/shippable/freebsd.sh

@@ -10,4 +10,5 @@ version="${args[1]}"
 target="posix/ci/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    --exclude "posix/ci/cloud/"

test/utils/shippable/linux.sh

@@ -9,4 +9,4 @@ image="ansible/ansible:${args[1]}"
 target="posix/ci/group${args[2]}/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \

test/utils/shippable/network.sh

@@ -2,7 +2,13 @@
 
 set -o pipefail
 
-ansible-test network-integration --explain 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
+# shellcheck disable=SC2086
+ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
+
+if [ "${COVERAGE}" ]; then
+    # when on-demand coverage is enabled, force tests to run for all network platforms
+    echo "coverage" > /tmp/network.txt
+fi
 
 target="network/ci/"
 
@@ -13,7 +19,7 @@ if [ -s /tmp/network.txt ]; then
     echo "Running network integration tests for multiple platforms concurrently."
 
     # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
         --platform vyos/1.1.0 \
         --platform ios/csr1000v \
 
@@ -22,6 +28,6 @@ else
     echo "Running network integration tests for a single platform only."
 
     # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
         --platform vyos/1.1.0
 fi

test/utils/shippable/osx.sh

@@ -10,5 +10,5 @@ version="${args[1]}"
 target="posix/ci/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \
-    ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    --exclude "posix/ci/cloud/" --remote-terminate always

test/utils/shippable/other.sh

@@ -10,13 +10,16 @@ retry.py pip install tox --disable-pip-version-check
 
 echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json
 
-ansible-test compile --failure-ok --color -v --junit --requirements --coverage
-ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage
-ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
+# shellcheck disable=SC2086
+ansible-test compile --failure-ok --color -v --junit --requirements --coverage ${CHANGED:+"$CHANGED"}
+# shellcheck disable=SC2086
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage ${CHANGED:+"$CHANGED"}
+# shellcheck disable=SC2086
+ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage ${CHANGED:+"$CHANGED"}
 
 rm test/results/bot/ansible-test-failure.json
 
 if find test/results/bot/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-    echo "One or more of the above ansible-test commands recorded at least one test failure."
+    echo "One or more of the above tests reported at least one failure."
     exit 1
 fi

test/utils/shippable/shippable.sh

@@ -23,11 +23,27 @@ pip list --disable-pip-version-check
 
 export PATH="test/runner:${PATH}"
 export PYTHONIOENCODING='utf-8'
-export COVERAGE="${COVERAGE:-}"
 
-# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty
-if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then
+if [ -n "${COVERAGE:-}" ]; then
+    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
     export COVERAGE="--coverage"
+elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
+    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
+    export COVERAGE="--coverage"
+else
+    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
+    export COVERAGE=""
+fi
+
+if [ -n "${COMPLETE:-}" ]; then
+    # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value
+    export CHANGED=""
+elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then
+    # disable change detection triggered by having 'ci_complete' in the latest commit message
+    export CHANGED=""
+else
+    # enable change detection (default behavior)
+    export CHANGED="--changed"
 fi
 
 # remove empty core/extras module directories from PRs created prior to the repo-merge

test/utils/shippable/units.sh

@@ -9,4 +9,5 @@ version="${args[1]}"
 
 retry.py pip install tox --disable-pip-version-check
 
-ansible-test units --color -v --tox --coverage --python "${version}"
+# shellcheck disable=SC2086
+ansible-test units --color -v --tox --python "${version}" --coverage ${CHANGED:+"$CHANGED"} \

test/utils/shippable/windows.sh

@@ -7,7 +7,8 @@ IFS='/:' read -ra args <<< "${TEST}"
 
 job="${args[1]}"
 
-ansible-test windows-integration --explain 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
+# shellcheck disable=SC2086
+ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
 
 if [ -s /tmp/windows.txt ]; then
     echo "Detected changes requiring integration tests specific to Windows:"
@@ -23,7 +24,7 @@ if [ -s /tmp/windows.txt ]; then
     target="windows/ci/"
 
     # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
         --windows 2008-SP2 \
         --windows 2008-R2_SP1 \
         --windows 2012-RTM \
@@ -36,6 +37,6 @@ else
     target="windows/ci/group${job}/"
 
     # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
         --windows 2012-R2_RTM
 fi