Run OS X and FreeBSD CI tests using groups. (#31254)

* Run OS X and FreeBSD CI tests using groups.
* Improve readability of Shippable jobs.

shippable.yml

@@ -2,67 +2,71 @@ language: python
 
 env:
   matrix:
-    - TEST=none
+    - T=none
 
 matrix:
   exclude:
-    - env: TEST=none
+    - env: T=none
   include:
-    - env: TEST=other
+    - env: T=other
 
-    - env: TEST=units/2.6
+    - env: T=units/2.6
-    - env: TEST=units/2.7
+    - env: T=units/2.7
-    - env: TEST=units/3.5
+    - env: T=units/3.5
-    - env: TEST=units/3.6
+    - env: T=units/3.6
 
-    - env: TEST=osx/10.11
+    - env: T=rhel/7.4
 
-    - env: TEST=freebsd/10.3-STABLE
+    - env: T=windows/1
-    - env: TEST=freebsd/11.0-STABLE
+    - env: T=windows/2
+    - env: T=windows/3
 
-    - env: TEST=rhel/7.4
+    - env: T=network
 
-    - env: TEST=windows/1
+    - env: T=osx/10.11/1
-    - env: TEST=windows/2
+    - env: T=freebsd/10.3-STABLE/1
-    - env: TEST=windows/3
+    - env: T=freebsd/11.0-STABLE/1
+    - env: T=linux/centos6/1
+    - env: T=linux/centos7/1
+    - env: T=linux/fedora24/1
+    - env: T=linux/fedora25/1
+    - env: T=linux/opensuse42.2/1
+    - env: T=linux/opensuse42.3/1
+    - env: T=linux/ubuntu1404/1
+    - env: T=linux/ubuntu1604/1
+    - env: T=linux/ubuntu1604py3/1
 
-    - env: TEST=network
+    - env: T=osx/10.11/2
+    - env: T=freebsd/10.3-STABLE/2
+    - env: T=freebsd/11.0-STABLE/2
+    - env: T=linux/centos6/2
+    - env: T=linux/centos7/2
+    - env: T=linux/fedora24/2
+    - env: T=linux/fedora25/2
+    - env: T=linux/opensuse42.2/2
+    - env: T=linux/opensuse42.3/2
+    - env: T=linux/ubuntu1404/2
+    - env: T=linux/ubuntu1604/2
+    - env: T=linux/ubuntu1604py3/2
 
-    - env: TEST=linux/centos6/1
+    - env: T=osx/10.11/3
-    - env: TEST=linux/centos7/1
+    - env: T=freebsd/10.3-STABLE/3
-    - env: TEST=linux/fedora24/1
+    - env: T=freebsd/11.0-STABLE/3
-    - env: TEST=linux/fedora25/1
+    - env: T=linux/centos6/3
-    - env: TEST=linux/opensuse42.2/1
+    - env: T=linux/centos7/3
-    - env: TEST=linux/opensuse42.3/1
+    - env: T=linux/fedora24/3
-    - env: TEST=linux/ubuntu1404/1
+    - env: T=linux/fedora25/3
-    - env: TEST=linux/ubuntu1604/1
+    - env: T=linux/opensuse42.2/3
-    - env: TEST=linux/ubuntu1604py3/1
+    - env: T=linux/opensuse42.3/3
+    - env: T=linux/ubuntu1404/3
+    - env: T=linux/ubuntu1604/3
+    - env: T=linux/ubuntu1604py3/3
 
-    - env: TEST=linux/centos6/2
+    - env: T=cloud/ubuntu1604/1
-    - env: TEST=linux/centos7/2
+    - env: T=cloud/ubuntu1604py3/1
-    - env: TEST=linux/fedora24/2
-    - env: TEST=linux/fedora25/2
-    - env: TEST=linux/opensuse42.2/2
-    - env: TEST=linux/opensuse42.3/2
-    - env: TEST=linux/ubuntu1404/2
-    - env: TEST=linux/ubuntu1604/2
-    - env: TEST=linux/ubuntu1604py3/2
 
-    - env: TEST=linux/centos6/3
+    - env: T=cloud/ubuntu1604/2
-    - env: TEST=linux/centos7/3
+    - env: T=cloud/ubuntu1604py3/2
-    - env: TEST=linux/fedora24/3
-    - env: TEST=linux/fedora25/3
-    - env: TEST=linux/opensuse42.2/3
-    - env: TEST=linux/opensuse42.3/3
-    - env: TEST=linux/ubuntu1404/3
-    - env: TEST=linux/ubuntu1604/3
-    - env: TEST=linux/ubuntu1604py3/3
-
-    - env: TEST=cloud/ubuntu1604/1
-    - env: TEST=cloud/ubuntu1604py3/1
-
-    - env: TEST=cloud/ubuntu1604/2
-    - env: TEST=cloud/ubuntu1604py3/2
 
 branches:
   except:
@@ -76,7 +80,7 @@ build:
     pull: false
     options: "--privileged=false --net=bridge"
   ci:
-    - test/utils/shippable/timing.sh test/utils/shippable/shippable.sh
+    - test/utils/shippable/timing.sh test/utils/shippable/shippable.sh $T
 
 integrations:
   notifications:

test/utils/shippable/cloud.sh

@@ -3,7 +3,7 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 image="ansible/ansible:${args[1]}"
 target="posix/ci/cloud/group${args[2]}/"

test/utils/shippable/freebsd.sh

@@ -3,12 +3,12 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 platform="${args[0]}"
 version="${args[1]}"
-target="posix/ci/"
+target="posix/ci/group${args[2]}/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-    --exclude "posix/ci/cloud/"
+     --remote "${platform}/${version}" --remote-terminate always

test/utils/shippable/linux.sh

@@ -3,10 +3,11 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 image="ansible/ansible:${args[1]}"
 target="posix/ci/group${args[2]}/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+     --docker "${image}"

test/utils/shippable/osx.sh

@@ -3,12 +3,12 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 platform="${args[0]}"
 version="${args[1]}"
-target="posix/ci/"
+target="posix/ci/group${args[2]}/"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-    --exclude "posix/ci/cloud/" --remote-terminate always
+     --remote "${platform}/${version}" --remote-terminate always

test/utils/shippable/rhel.sh

@@ -3,7 +3,7 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 platform="${args[0]}"
 version="${args[1]}"

test/utils/shippable/shippable.sh

@@ -3,10 +3,12 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 script="${args[0]}"
 
+test="$1"
+
 docker images ansible/ansible
 docker ps
 
@@ -58,7 +60,7 @@ function cleanup
 {
     if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
         # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
-        if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ] && [ "${TEST}" == "other" ]; then
+        if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "other" ]; then
             stub="--stub"
         else
             stub=""
@@ -79,7 +81,7 @@ function cleanup
                 bash <(curl -s https://codecov.io/bash) \
                     -f "${file}" \
                     -F "${flags}" \
-                    -n "${TEST}" \
+                    -n "${test}" \
                     -t 83cd8957-dc76-488c-9ada-210dcea51633 \
                     -X coveragepy \
                     -X gcov \
@@ -98,4 +100,4 @@ function cleanup
 
 trap cleanup EXIT
 
-"test/utils/shippable/${script}.sh"
+"test/utils/shippable/${script}.sh" "${test}"

test/utils/shippable/timing.sh

@@ -2,4 +2,4 @@
 
 set -o pipefail
 
-"$1" 2>&1 | gawk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush(); }'
+"$@" 2>&1 | gawk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush(); }'

test/utils/shippable/units.sh

@@ -3,7 +3,7 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 version="${args[1]}"
 

test/utils/shippable/windows.sh

@@ -3,7 +3,7 @@
 set -o pipefail
 
 declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
+IFS='/:' read -ra args <<< "$1"
 
 target="windows/ci/group${args[1]}/"