[WIP] Removed windows dep and unify shippable.sh (#38)

* Removed windows dep and unify shippable.sh

* Removed uneeded -p

* Fix chmod.

* Update tests/utils/shippable/shippable.sh

Co-Authored-By: John R Barker <john@johnrbarker.com>

* Distribute sanity tests into groups.

* Remove docs-build test, which doesn't yet exist for collections.

* Remove package-data.

* Remove unsupported option.

* Fix paths.

Co-authored-by: Jordan Borean <jborean93@gmail.com>
Co-authored-by: John R Barker <john@johnrbarker.com>

tests/integration/targets/postgresql_ext/tasks/postgresql_ext_initial.yml

@@ -1,9 +1,6 @@
 - name: postgresql_ext - install postgis on Linux
   package: name=postgis state=present
   when: ansible_os_family != "Windows"
-- name: postgresql_ext - install postgis on Windows
-  when: ansible_os_family == "Windows"
-  ansible.windows.win_package: name=postgis state=present
 - name: postgresql_ext - create schema schema1
   become_user: '{{ pg_user }}'
   become: true

tests/integration/targets/setup_remote_tmp_dir/tasks/windows-cleanup.yml

@@ -1,4 +0,0 @@
-- name: delete temporary directory (windows)
-  ansible.windows.win_file:
-    path: '{{ remote_tmp_dir }}'
-    state: absent

tests/integration/targets/setup_remote_tmp_dir/tasks/windows.yml

@@ -1,10 +0,0 @@
-- name: create temporary directory
-  register: remote_tmp_dir
-  notify:
-  - delete temporary directory (windows)
-  ansible.windows.win_tempfile:
-    state: directory
-    suffix: .test
-- name: record temporary directory
-  set_fact:
-    remote_tmp_dir: '{{ remote_tmp_dir.path }}'

tests/requirements.yml

@@ -1,5 +1,4 @@
 integration_tests_dependencies:
-- ansible.windows
 - ansible.posix
 - community.crypto
 - ansible.netcommon

tests/utils/shippable/sanity.sh

@@ -2,6 +2,37 @@
 
 set -o pipefail -eux
 
+declare -a args
+IFS='/:' read -ra args <<< "$1"
+
+group="${args[1]}"
+
+if [ "${BASE_BRANCH:-}" ]; then
+    base_branch="origin/${BASE_BRANCH}"
+else
+    base_branch=""
+fi
+
+case "${group}" in
+    1) options=(--skip-test pylint --skip-test ansible-doc --skip-test validate-modules) ;;
+    2) options=(                   --test      ansible-doc                             ) ;;
+    3) options=(                                                --test validate-modules) ;;
+    4) options=(--test pylint --exclude tests/unit/ --exclude plugins/module_utils/) ;;
+    5) options=(--test pylint           tests/unit/           plugins/module_utils/) ;;
+esac
+
+# allow collection migration sanity tests for groups 3 and 4 to pass without updating this script during migration
+network_path="lib/ansible/modules/network/"
+
+if [ -d "${network_path}" ]; then
+    if [ "${group}" -eq 3 ]; then
+        options+=(--exclude "${network_path}")
+    elif [ "${group}" -eq 4 ]; then
+        options+=("${network_path}")
+    fi
+fi
+
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-    --docker \
+    --docker --base-branch "${base_branch}" \
+    "${options[@]}" --allow-disabled

tests/utils/shippable/shippable.sh

@@ -29,8 +29,36 @@ python -V
 command -v pip
 pip --version
 pip list --disable-pip-version-check
+pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
+
+export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
+SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
+TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general"
+mkdir -p "${TEST_DIR}"
+cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
+cd "${TEST_DIR}"
+
+# STAR: HACK install dependencies
+ansible-galaxy -vvv collection install ansible.posix
+ansible-galaxy -vvv collection install community.crypto
+ansible-galaxy -vvv collection install ansible.netcommon
+
+# unit tests
+ansible-galaxy -vvv collection install community.kubernetes
+ansible-galaxy -vvv collection install netbox.netbox
+ansible-galaxy -vvv collection install netapp.ontap
+ansible-galaxy -vvv collection install cisco.meraki
+ansible-galaxy -vvv collection install fortinet.fortios
+ansible-galaxy -vvv collection install junipernetworks.junos
+ansible-galaxy -vvv collection install cisco.aci
+ansible-galaxy -vvv collection install google.cloud
+ansible-galaxy -vvv collection install community.kubernetes
+ansible-galaxy -vvv collection install f5networks.f5_modules
+
+# Needed until https://github.com/ansible/ansible/issues/68415 is fixed:
+chmod -R a+rX "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections"
+# END: HACK
 
-export PATH="${PWD}/bin:${PATH}"
 export PYTHONIOENCODING='utf-8'
 
 if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then
@@ -68,47 +96,81 @@ else
     export UNSTABLE=""
 fi
 
-virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+# remove empty core/extras module directories from PRs created prior to the repo-merge
-set +ux
+find plugins -type d -empty -print -delete
-. ~/ansible-venv/bin/activate
-set -ux
-
-#pip install ansible==2.9.0 --disable-pip-version-check
-pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
-
-COLLECTION_DIR="${HOME}/.ansible/ansible_collections/"
-TEST_DIR="${COLLECTION_DIR}/community/general"
-mkdir -p "${TEST_DIR}"
-cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
-cd "${TEST_DIR}"
 
 function cleanup
 {
+    # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy
+    if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then
+        stub="--stub"
+        # trigger coverage reporting for stubs even if no other coverage data exists
+        mkdir -p tests/output/coverage/
+    else
+        stub=""
+    fi
+
     if [ -d tests/output/coverage/ ]; then
         if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-            # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
+            process_coverage='yes'  # process existing coverage files
-            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then
+        elif [ "${stub}" ]; then
-                stub="--stub"
+            process_coverage='yes'  # process coverage when stubs are enabled
-            else
+        else
-                stub=""
+            process_coverage=''
-            fi
+        fi
+
+        if [ "${process_coverage}" ]; then
+            # use python 3.7 for coverage to avoid running out of memory during coverage xml processing
+            # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job
+            virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+            set +ux
+            . ~/ansible-venv/bin/activate
+            set -ux
 
             # shellcheck disable=SC2086
-            ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
+            ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"}
-            cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/
+            cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/"
+
+            # analyze and capture code coverage aggregated by integration test target
+            ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json"
+
+            # upload coverage report to codecov.io only when using complete on-demand coverage
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
+                for file in tests/output/reports/coverage=*.xml; do
+                    flags="${file##*/coverage=}"
+                    flags="${flags%-powershell.xml}"
+                    flags="${flags%.xml}"
+                    # remove numbered component from stub files when converting to tags
+                    flags="${flags//stub-[0-9]*/stub}"
+                    flags="${flags//=/,}"
+                    flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                    bash <(curl -s https://codecov.io/bash) \
+                        -f "${file}" \
+                        -F "${flags}" \
+                        -n "${test}" \
+                        -t 20636cf5-4d6a-4b9a-8d2d-6f22ebbaa752 \
+                        -X coveragepy \
+                        -X gcov \
+                        -X fix \
+                        -X search \
+                        -X xcode \
+                    || echo "Failed to upload code coverage report to codecov.io: ${file}"
+                done
+            fi
         fi
     fi
 
     if [ -d  tests/output/junit/ ]; then
-      cp -aT tests/output/junit/ shippable/testresults/
+      cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d tests/output/data/ ]; then
-      cp -a tests/output/data/ shippable/testresults/
+      cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d  tests/output/bot/ ]; then
-      cp -aT tests/output/bot/ shippable/testresults/
+      cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 }
 
@@ -117,41 +179,9 @@ trap cleanup EXIT
 if [[ "${COVERAGE:-}" == "--coverage" ]]; then
     timeout=60
 else
-    timeout=45
+    timeout=50
 fi
 
-# STAR: HACK install dependencies
-(
-#ansible.windows doesn't install from Galaxy
-mkdir /tmp/collection_deps
-git clone https://github.com/ansible-collections/ansible.windows.git /tmp/collection_deps/ansible.windows
-cd /tmp/collection_deps/ansible.windows
-ansible-galaxy collection build
-ansible-galaxy collection install /tmp/collection_deps/ansible.windows/ansible-windows* -p "${COLLECTION_DIR}"
-)
-
-
-#ansible-galaxy collection install ansible.windows -p "${COLLECTION_DIR}"
-ansible-galaxy collection install ansible.posix -p "${COLLECTION_DIR}"
-ansible-galaxy collection install community.crypto -p "${COLLECTION_DIR}"
-ansible-galaxy collection install ansible.netcommon -p "${COLLECTION_DIR}"
-
-# unit tests
-ansible-galaxy collection install community.kubernetes -p "${COLLECTION_DIR}"
-ansible-galaxy collection install netbox.netbox -p "${COLLECTION_DIR}"
-ansible-galaxy collection install netapp.ontap -p "${COLLECTION_DIR}"
-ansible-galaxy collection install cisco.meraki -p "${COLLECTION_DIR}"
-ansible-galaxy collection install fortinet.fortios -p "${COLLECTION_DIR}"
-ansible-galaxy collection install junipernetworks.junos -p "${COLLECTION_DIR}"
-ansible-galaxy collection install cisco.aci -p "${COLLECTION_DIR}"
-ansible-galaxy collection install google.cloud -p "${COLLECTION_DIR}"
-ansible-galaxy collection install community.kubernetes -p "${COLLECTION_DIR}"
-ansible-galaxy collection install f5networks.f5_modules -p "${COLLECTION_DIR}"
-
-chmod -R a+rX "${COLLECTION_DIR}"
-
-# END: HACK
-
 ansible-test env --dump --show --timeout "${timeout}" --color -v
 
 "tests/utils/shippable/check_matrix.py"