From f76afab6e54a65cfa8db2d8dc330a72f3eb430ad Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 19 Oct 2017 13:37:22 -0700 Subject: [PATCH] Add new default Docker container for ansible-test. (#31944) * Add new default Docker container for ansible-test. * Update ansible-test change classification. * Update list of disabled pylint rules. * Fix pylint issues with ansible-test. --- test/runner/.dockerignore | 3 ++ test/runner/Dockerfile | 39 ++++++++++++++++++++++++ test/runner/docker/deadsnakes.list | 2 ++ test/runner/docker/requirements.sh | 23 ++++++++++++++ test/runner/lib/classification.py | 29 ++++++++++++++++++ test/runner/lib/sanity/__init__.py | 2 +- test/runner/requirements/constraints.txt | 3 +- test/runner/requirements/sanity.txt | 2 +- test/sanity/pylint/disable.txt | 15 +++++++++ 9 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 test/runner/.dockerignore create mode 100644 test/runner/Dockerfile create mode 100644 test/runner/docker/deadsnakes.list create mode 100755 test/runner/docker/requirements.sh diff --git a/test/runner/.dockerignore b/test/runner/.dockerignore new file mode 100644 index 0000000000..c1e8015167 --- /dev/null +++ b/test/runner/.dockerignore @@ -0,0 +1,3 @@ +* +!docker +!requirements diff --git a/test/runner/Dockerfile b/test/runner/Dockerfile new file mode 100644 index 0000000000..d08e7f8616 --- /dev/null +++ b/test/runner/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:16.04 + +COPY docker/deadsnakes.list /etc/apt/sources.list.d/deadsnakes.list + +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 + +RUN apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gcc \ + git \ + libffi-dev \ + libxml2-dev \ + libxslt1-dev \ + locales \ + make \ + python2.6-dev \ + python2.7-dev \ + python3.5-dev \ + python3.6-dev \ + shellcheck \ + && \ + apt-get clean + +RUN rm /etc/apt/apt.conf.d/docker-clean +RUN locale-gen en_US.UTF-8 +VOLUME /sys/fs/cgroup /run/lock /run /tmp + +ADD https://bootstrap.pypa.io/get-pip.py /tmp/get-pip.py + +COPY requirements/*.txt /tmp/requirements/ +COPY docker/requirements.sh /tmp/ +RUN cd /tmp/requirements && /tmp/requirements.sh + +RUN ln -s python2.7 /usr/bin/python2 +RUN ln -s python3.6 /usr/bin/python3 +RUN ln -s python3 /usr/bin/python + +ENV container=docker +CMD ["/sbin/init"] diff --git a/test/runner/docker/deadsnakes.list b/test/runner/docker/deadsnakes.list new file mode 100644 index 0000000000..32a1fbc14f --- /dev/null +++ b/test/runner/docker/deadsnakes.list @@ -0,0 +1,2 @@ +deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main +deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main diff --git a/test/runner/docker/requirements.sh b/test/runner/docker/requirements.sh new file mode 100755 index 0000000000..f16e1b7e98 --- /dev/null +++ b/test/runner/docker/requirements.sh @@ -0,0 +1,23 @@ +#!/bin/bash -eu + +python_versions=( + 2.6 + 2.7 + 3.5 + 3.6 +) + +requirements=() + +for requirement in *.txt; do + if [ "${requirement}" != "constraints.txt" ]; then + requirements+=("-r" "${requirement}") + fi +done + +for python_version in "${python_versions[@]}"; do + set -x + "python${python_version}" /tmp/get-pip.py -c constraints.txt + "pip${python_version}" install --disable-pip-version-check -c constraints.txt "${requirements[@]}" + set +x +done diff --git a/test/runner/lib/classification.py b/test/runner/lib/classification.py index d63fc1c290..f7c89bd24e 100644 --- a/test/runner/lib/classification.py +++ b/test/runner/lib/classification.py @@ -461,6 +461,9 @@ class PathMapper(object): test_path = os.path.dirname(test_path) + if path.startswith('test/runner/docker/'): + return minimal # not used by tests, only used to build the default container + if path.startswith('test/runner/lib/cloud/'): cloud_target = 'cloud/%s/' % name @@ -476,6 +479,32 @@ class PathMapper(object): 'sanity': 'all', # test infrastructure, run all sanity checks } + if path.startswith('test/runner/requirements/'): + if name in ( + 'integration', + 'network-integration', + 'windows-integration', + ): + return { + name: self.integration_all_target, + } + + if name in ( + 'sanity', + 'units', + ): + return { + name: 'all', + } + + if name.startswith('integration.cloud.'): + cloud_target = 'cloud/%s/' % name.split('.')[2] + + if cloud_target in self.integration_targets_by_alias: + return { + 'integration': cloud_target, + } + if path.startswith('test/runner/'): return all_tests(self.args) # test infrastructure, run all tests diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py index 0589b1bddd..0b42aefb25 100644 --- a/test/runner/lib/sanity/__init__.py +++ b/test/runner/lib/sanity/__init__.py @@ -83,7 +83,7 @@ def command_sanity(args): if isinstance(test, SanityMultipleVersion): versions = SUPPORTED_PYTHON_VERSIONS else: - versions = None, + versions = (None,) for version in versions: if args.python and version and version != args.python: diff --git a/test/runner/requirements/constraints.txt b/test/runner/requirements/constraints.txt index b229cc763a..ef5da0f6a8 100644 --- a/test/runner/requirements/constraints.txt +++ b/test/runner/requirements/constraints.txt @@ -1,6 +1,7 @@ coverage >= 4.2, != 4.3.2 # features in 4.2+ required, avoid known bug in 4.3.2 on python 2.6 pywinrm >= 0.2.1 # 0.1.1 required, but 0.2.1 provides better performance -pylint >= 1.5.3, < 1.7.0 # 1.4.1 adds JSON output, but 1.5.3 fixes bugs related to JSON output +pylint == 1.7.4 ; python_version >= '3.5' # versions before 1.7.1 hang or fail to install on python 3.x +pylint == 1.6.5 ; python_version <= '2.7' # versions after 1.6.5 hang or fail during test on python 2.x sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7 or later wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later yamllint != 1.8.0 ; python_version < '2.7' # yamllint 1.8.0 requires python 2.7+ while earlier/later versions do not diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt index df8c7c0713..34d6e07f47 100644 --- a/test/runner/requirements/sanity.txt +++ b/test/runner/requirements/sanity.txt @@ -3,7 +3,7 @@ jinja2 mock paramiko pycodestyle -pylint +pylint ; python_version >= '2.7' # pylint 1.5.3+ is required for non-buggy JSON output, but 1.4+ requires python 2.7+ pytest rstcheck sphinx diff --git a/test/sanity/pylint/disable.txt b/test/sanity/pylint/disable.txt index c014296007..2ffed769c8 100644 --- a/test/sanity/pylint/disable.txt +++ b/test/sanity/pylint/disable.txt @@ -13,7 +13,9 @@ blacklisted-name broad-except cell-var-from-loop consider-iterating-dictionary +consider-merging-isinstance consider-using-enumerate +consider-using-ternary deprecated-lambda deprecated-method deprecated-module @@ -29,11 +31,16 @@ global-variable-undefined import-error import-self invalid-name +invalid-sequence-index +invalid-unary-operand-type +len-as-condition line-too-long +literal-comparison locally-disabled method-hidden misplaced-comparison-constant missing-docstring +no-else-return no-init no-member no-name-in-module @@ -47,11 +54,14 @@ old-style-class pointless-statement pointless-string-statement protected-access +pylint +redefined-argument-from-local redefined-builtin redefined-outer-name redefined-variable-type reimported relative-import +signature-differs simplifiable-if-statement super-init-not-called superfluous-parens @@ -68,17 +78,22 @@ too-many-nested-blocks too-many-public-methods too-many-return-statements too-many-statements +trailing-comma-tuple +unbalanced-tuple-unpacking undefined-loop-variable ungrouped-imports unidiomatic-typecheck unneeded-not unsubscriptable-object +unsupported-assignment-operation +unsupported-delete-operation unsupported-membership-test unused-argument unused-import unused-variable unused-wildcard-import used-before-assignment +useless-super-delegation wildcard-import wrong-import-order wrong-import-position