Switch tests to pytest and ansible-test.

- Replace nose usage with pytest.
- Remove legacy Shippable integration.sh.
- Update Makefile to use pytest and ansible-test.
- Convert most yield unit tests to pytest parametrize.

Makefile

@@ -98,22 +98,30 @@ RPMNVR = "$(NAME)-$(VERSION)-$(RPMRELEASE)$(RPMDIST)"
 MOCK_BIN ?= mock
 MOCK_CFG ?=
 
-NOSETESTS ?= nosetests
+# ansible-test parameters
+ANSIBLE_TEST ?= test/runner/ansible-test
+TEST_FLAGS ?=
 
-NOSETESTS3 ?= nosetests-3.5
+# ansible-test units parameters (make test / make test-py3)
+PYTHON_VERSION ?= $(shell python2 -c 'import sys; print("%s.%s" % sys.version_info[:2])')
+PYTHON3_VERSION ?= $(shell python3 -c 'import sys; print("%s.%s" % sys.version_info[:2])')
+
+# ansible-test integration parameters (make integration)
+IMAGE ?= centos7
+TARGET ?=
 
 ########################################################
 
 all: clean python
 
 tests:
-	PYTHONPATH=./lib $(NOSETESTS) -d -w test/units -v --with-coverage --cover-package=ansible --cover-branches --cover-erase -e test_os_server
+	$(ANSIBLE_TEST) units -v --python $(PYTHON_VERSION) $(TEST_FLAGS)
 
 tests-py3:
-	PYTHONPATH=./lib $(NOSETESTS3) -d -w test/units -v --with-coverage --cover-package=ansible --cover-branches --cover-erase -e test_os_server
+	$(ANSIBLE_TEST) units -v --python $(PYTHON3_VERSION) $(TEST_FLAGS)
 
 integration:
-	test/utils/shippable/integration.sh
+	$(ANSIBLE_TEST) integration -v --docker $(IMAGE) $(TARGET) $(TEST_FLAGS)
 
 authors:
 	sh hacking/authors.sh

hacking/README.md

@@ -17,7 +17,7 @@ and do not wish to install them from your operating system package manager, you
 can install them from pip
 
     $ easy_install pip               # if pip is not already available
-    $ pip install pyyaml jinja2 nose passlib pycrypto
+    $ pip install pyyaml jinja2 nose pytest passlib pycrypto
 
 From there, follow ansible instructions on docs.ansible.com as normal.
 

lib/ansible/test-requirements.txt

@@ -14,3 +14,4 @@ six
 unittest2
 mock
 nose
+pytest

test/README.md

@@ -12,7 +12,7 @@ mock interfaces rather than producing side effects.
 
 Playbook engine code is better suited for integration tests.
 
-Requirements: `sudo pip install paramiko PyYAML jinja2 httplib2 passlib nose mock`
+Requirements: `sudo pip install paramiko PyYAML jinja2 httplib2 passlib nose pytest mock`
 
 integration
 -----------

test/units/module_utils/test_database.py

@@ -1,43 +1,13 @@
-import collections
-import os
-import re
-
-
-from nose.tools import eq_
-try:
-    from nose.tools import assert_raises_regexp
-except ImportError:
-    from ansible.compat.six import string_types
-    # Python < 2.7
-    def assert_raises_regexp(expected, regexp, callable, *a, **kw):
-        try:
-            callable(*a, **kw)
-        except expected as e:
-            if isinstance(regexp, string_types):
-                regexp = re.compile(regexp)
-            if not regexp.search(str(e)):
-                raise Exception('"%s" does not match "%s"' %
-                                (regexp.pattern, str(e)))
-        else:
-            if hasattr(expected,'__name__'): excName = expected.__name__
-            else: excName = str(expected)
-            raise AssertionError("%s not raised" % excName)
-
-from ansible.compat.tests import mock
+import pytest
 
 from ansible.module_utils.database import (
     pg_quote_identifier,
     SQLParseError,
 )
 
-
-# Note: Using nose's generator test cases here so we can't inherit from
-# unittest.TestCase
-class TestQuotePgIdentifier(object):
-
-    # These are all valid strings
-    # The results are based on interpreting the identifier as a table name
-    valid = {
+# These are all valid strings
+# The results are based on interpreting the identifier as a table name
+VALID = {
     # User quoted
     '"public.table"': '"public.table"',
     '"public"."table"': '"public"."table"',
@@ -74,9 +44,9 @@ class TestQuotePgIdentifier(object):
     # These are valid but maybe not what the user intended
     '."table"': '".""table"""',
     'table.': '"table."',
-    }
+}
 
-    invalid = {
+INVALID = {
     ('test.too.many.dots', 'table'): 'PostgreSQL does not support table with more than 3 dots',
     ('"test.too".many.dots', 'database'): 'PostgreSQL does not support database with more than 1 dots',
     ('test.too."many.dots"', 'database'): 'PostgreSQL does not support database with more than 1 dots',
@@ -86,36 +56,45 @@ class TestQuotePgIdentifier(object):
     ('"test"."too"."many"."dots"."for"."column"', 'column'): "PostgreSQL does not support column with more than 4 dots",
     ('"table "invalid" double quote"', 'table'): 'User escaped identifiers must escape extra quotes',
     ('"schema "invalid"""."table "invalid"', 'table'): 'User escaped identifiers must escape extra quotes',
-        ('"schema."table"','table'): 'User escaped identifiers must escape extra quotes',
+    ('"schema."table"', 'table'): 'User escaped identifiers must escape extra quotes',
     ('"schema".', 'table'): 'Identifier name unspecified or unquoted trailing dot',
-    }
+}
 
-    def check_valid_quotes(self, identifier, quoted_identifier):
-        eq_(pg_quote_identifier(identifier, 'table'), quoted_identifier)
+HOW_MANY_DOTS = (
+    ('role', 'role', '"role"',
+     'PostgreSQL does not support role with more than 1 dots'),
+    ('db', 'database', '"db"',
+     'PostgreSQL does not support database with more than 1 dots'),
+    ('db.schema', 'schema', '"db"."schema"',
+     'PostgreSQL does not support schema with more than 2 dots'),
+    ('db.schema.table', 'table', '"db"."schema"."table"',
+     'PostgreSQL does not support table with more than 3 dots'),
+    ('db.schema.table.column', 'column', '"db"."schema"."table"."column"',
+     'PostgreSQL does not support column with more than 4 dots'),
+)
 
-    def test_valid_quotes(self):
-        for identifier in self.valid:
-            yield self.check_valid_quotes, identifier, self.valid[identifier]
+VALID_QUOTES = ((test, VALID[test]) for test in VALID)
+INVALID_QUOTES = ((test[0], test[1], INVALID[test]) for test in INVALID)
 
-    def check_invalid_quotes(self, identifier, id_type, msg):
-        assert_raises_regexp(SQLParseError, msg, pg_quote_identifier, *(identifier, id_type))
 
-    def test_invalid_quotes(self):
-        for test in self.invalid:
-            yield self.check_invalid_quotes, test[0], test[1], self.invalid[test]
+@pytest.mark.parametrize("identifier, quoted_identifier", VALID_QUOTES)
+def test_valid_quotes(identifier, quoted_identifier):
+    assert pg_quote_identifier(identifier, 'table') == quoted_identifier
 
-    def test_how_many_dots(self):
-        eq_(pg_quote_identifier('role', 'role'), '"role"')
-        assert_raises_regexp(SQLParseError, "PostgreSQL does not support role with more than 1 dots", pg_quote_identifier, *('role.more', 'role'))
 
-        eq_(pg_quote_identifier('db', 'database'), '"db"')
-        assert_raises_regexp(SQLParseError, "PostgreSQL does not support database with more than 1 dots", pg_quote_identifier, *('db.more', 'database'))
+@pytest.mark.parametrize("identifier, id_type, msg", INVALID_QUOTES)
+def test_invalid_quotes(identifier, id_type, msg):
+    with pytest.raises(SQLParseError) as ex:
+        pg_quote_identifier(identifier, id_type)
 
-        eq_(pg_quote_identifier('db.schema', 'schema'), '"db"."schema"')
-        assert_raises_regexp(SQLParseError, "PostgreSQL does not support schema with more than 2 dots", pg_quote_identifier, *('db.schema.more', 'schema'))
+    ex.match(msg)
 
-        eq_(pg_quote_identifier('db.schema.table', 'table'), '"db"."schema"."table"')
-        assert_raises_regexp(SQLParseError, "PostgreSQL does not support table with more than 3 dots", pg_quote_identifier, *('db.schema.table.more', 'table'))
 
-        eq_(pg_quote_identifier('db.schema.table.column', 'column'), '"db"."schema"."table"."column"')
-        assert_raises_regexp(SQLParseError, "PostgreSQL does not support column with more than 4 dots", pg_quote_identifier, *('db.schema.table.column.more', 'column'))
+@pytest.mark.parametrize("identifier, id_type, quoted_identifier, msg", HOW_MANY_DOTS)
+def test_how_many_dots(identifier, id_type, quoted_identifier, msg):
+    assert pg_quote_identifier(identifier, id_type) == quoted_identifier
+
+    with pytest.raises(SQLParseError) as ex:
+        pg_quote_identifier('%s.more' % identifier, id_type)
+
+    ex.match(msg)

test/units/parsing/test_splitter.py

@@ -20,16 +20,11 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from nose import tools
-from ansible.compat.tests import unittest
-
 from ansible.parsing.splitter import split_args, parse_kv
 
+import pytest
 
-# Tests using nose's test generators cannot use unittest base class.
-# http://nose.readthedocs.org/en/latest/writing_tests.html#test-generators
-class TestSplitter_Gen:
-    SPLIT_DATA = (
+SPLIT_DATA = (
     (u'a',
         [u'a'],
         {u'_raw_params': u'a'}),
@@ -96,20 +91,17 @@ class TestSplitter_Gen:
     (u'a={{ foo | some_filter(\' \', " ") }} b=bar',
         [u'a={{ foo | some_filter(\' \', " ") }}', u'b=bar'],
         {u'a': u'{{ foo | some_filter(\' \', " ") }}', u'b': u'bar'}),
-            )
+)
 
-    def check_split_args(self, args, expected):
-        tools.eq_(split_args(args), expected)
+SPLIT_ARGS = ((test[0], test[1]) for test in SPLIT_DATA)
+PARSE_KV = ((test[0], test[2]) for test in SPLIT_DATA)
 
-    def test_split_args(self):
-        for datapoint in self.SPLIT_DATA:
-            yield self.check_split_args, datapoint[0], datapoint[1]
 
-    def check_parse_kv(self, args, expected):
-        tools.eq_(parse_kv(args), expected)
+@pytest.mark.parametrize("args, expected", SPLIT_ARGS)
+def test_split_args(args, expected):
+    assert split_args(args) == expected
 
-    def test_parse_kv(self):
-        for datapoint in self.SPLIT_DATA:
-            try:
-                yield self.check_parse_kv, datapoint[0], datapoint[2]
-            except: pass
+
+@pytest.mark.parametrize("args, expected", PARSE_KV)
+def test_parse_kv(args, expected):
+    assert parse_kv(args) == expected

test/units/parsing/test_unquote.py

@@ -20,16 +20,11 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from nose import tools
-from ansible.compat.tests import unittest
-
 from ansible.parsing.quoting import unquote
 
+import pytest
 
-# Tests using nose's test generators cannot use unittest base class.
-# http://nose.readthedocs.org/en/latest/writing_tests.html#test-generators
-class TestUnquote:
-    UNQUOTE_DATA = (
+UNQUOTE_DATA = (
     (u'1', u'1'),
     (u'\'1\'', u'1'),
     (u'"1"', u'1'),
@@ -48,11 +43,9 @@ class TestUnquote:
     (u'\'1\'\'2\'', u'1\'\'2'),
     (u'"1" 2 "3"', u'1" 2 "3'),
     (u'"1"\'2\'"3"', u'1"\'2\'"3'),
-            )
+)
 
-    def check_unquote(self, quoted, expected):
-        tools.eq_(unquote(quoted), expected)
 
-    def test_unquote(self):
-        for datapoint in self.UNQUOTE_DATA:
-            yield self.check_unquote, datapoint[0], datapoint[1]
+@pytest.mark.parametrize("quoted, expected", UNQUOTE_DATA)
+def test_unquote(quoted, expected):
+    assert unquote(quoted) == expected

test/utils/shippable/integration-requirements.txt

@@ -1 +0,0 @@
-jmespath

test/utils/shippable/integration.sh

@@ -1,140 +0,0 @@
-#!/bin/bash -eux
-
-source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))")
-
-test_image="${IMAGE:-ansible/ansible:centos7}"
-test_privileged="${PRIVILEGED:-false}"
-test_flags="${TEST_FLAGS:-}"
-test_target="${TARGET:-all}"
-test_ansible_dir="${TEST_ANSIBLE_DIR:-/root/ansible}"
-test_python3="${PYTHON3:-}"
-
-http_image="${HTTP_IMAGE:-ansible/ansible:httptester}"
-
-# Keep the docker containers after tests complete.
-# The default behavior is to always remove the containers.
-# Set to "onfailure" to keep the containers only on test failure.
-# Any other non-empty value will always keep the containers.
-keep_containers="${KEEP_CONTAINERS:-}"
-
-# Run the tests directly from the source directory shared with the container.
-# The default behavior is to run the tests on a copy of the source.
-# Copying the source isolates changes to the source between host and container.
-# Set to any non-empty value to share the source.
-share_source="${SHARE_SOURCE:-}"
-
-# Force ansible color output by default.
-# To disable color force mode use FORCE_COLOR=0
-force_color="${FORCE_COLOR:-1}"
-
-if [ "${SHIPPABLE_BUILD_DIR:-}" ]; then
-    host_shared_dir="/home/shippable/cache/build-${BUILD_NUMBER}"
-    controller_shared_dir="/home/shippable/cache/build-${BUILD_NUMBER}"
-    share_source=1
-    test_privileged=false # temporarily disabled to troubleshoot performance issues
-else
-    host_shared_dir="${source_root}"
-    controller_shared_dir=""
-fi
-
-if [ -z "${share_source}" ]; then
-    test_shared_dir="/shared"
-else
-    test_shared_dir="${test_ansible_dir}"
-fi
-
-container_id=
-httptester_id=
-tests_completed=
-
-function show_environment
-{
-    docker ps
-
-    if [ -d /home/shippable/cache ]; then
-        ls -l /home/shippable/cache
-    fi
-}
-
-function cleanup
-{
-    if [ "${controller_shared_dir}" ]; then
-        cp -av "${controller_shared_dir}/shippable" "${SHIPPABLE_BUILD_DIR}"
-        rm -rf "${controller_shared_dir}"
-    fi
-
-    if [ "${keep_containers}" == "onfailure" ] && [ "${tests_completed}" != "" ]; then
-        keep_containers=
-    fi
-
-    if [ "${keep_containers}" == "" ]; then
-        if [ "${container_id}" ]; then
-            docker rm -f "${container_id}"
-        fi
-
-        if [ "${httptester_id}" ]; then
-            docker rm -f "${httptester_id}"
-        fi
-    fi
-
-    show_environment
-}
-
-trap cleanup EXIT INT TERM
-docker images ansible/ansible
-show_environment
-
-if [ "${controller_shared_dir}" ]; then
-    cp -a "${SHIPPABLE_BUILD_DIR}" "${controller_shared_dir}"
-fi
-
-httptester_id=$(docker run -d "${http_image}")
-container_id=$(docker run -d \
-    --env "ANSIBLE_FORCE_COLOR=${force_color}" \
-    -v "/sys/fs/cgroup:/sys/fs/cgroup:ro" \
-    -v "${host_shared_dir}:${test_shared_dir}" \
-    --link="${httptester_id}:ansible.http.tests" \
-    --link="${httptester_id}:sni1.ansible.http.tests" \
-    --link="${httptester_id}:sni2.ansible.http.tests" \
-    --link="${httptester_id}:fail.ansible.http.tests" \
-    --privileged="${test_privileged}" \
-    "${test_image}")
-
-show_environment
-
-skip=
-
-if [ "${test_python3}" ]; then
-    docker exec "${container_id}" ln -s /usr/bin/python3 /usr/bin/python
-    docker exec "${container_id}" ln -s /usr/bin/pip3 /usr/bin/pip
-
-    skip+=",$(tr '\n' ',' < "${source_root}/test/utils/shippable/python3-test-tag-blacklist.txt")"
-fi
-
-if [ "${test_privileged}" = 'false' ]; then
-    skip+=",needs_privileged"
-fi
-
-if [ "${skip}" ]; then
-    test_flags="--skip-tags ${skip} ${test_flags}"
-fi
-
-if [ -z "${share_source}" ]; then
-    docker exec "${container_id}" cp -a "${test_shared_dir}" "${test_ansible_dir}"
-fi
-
-docker exec "${container_id}" \
-    pip install -r "${test_ansible_dir}/test/utils/shippable/integration-requirements.txt" --upgrade
-
-if [ "${test_python3}" ]; then
-    docker exec "${container_id}" sed -i -f \
-        "${test_ansible_dir}/test/utils/shippable/python3-test-target-blacklist.txt" \
-        "${test_ansible_dir}/test/integration/Makefile"
-fi
-
-docker exec "${container_id}" mkdir -p "${test_shared_dir}/shippable/testresults"
-docker exec "${container_id}" /bin/sh -c "cd '${test_ansible_dir}' && . hacking/env-setup && cd test/integration && \
-    JUNIT_OUTPUT_DIR='${test_shared_dir}/shippable/testresults' ANSIBLE_CALLBACK_WHITELIST=junit \
-    HTTPTESTER=1 TEST_FLAGS='${test_flags}' LC_ALL=en_US.utf-8 make ${test_target}"
-
-tests_completed=1

test/utils/shippable/python3-test-tag-blacklist.txt

@@ -1,2 +0,0 @@
-test_hg
-test_service

test/utils/tox/requirements-py3.txt

@@ -13,3 +13,4 @@ python3-memcached
 python-systemd
 botocore
 boto3
+pytest

test/utils/tox/requirements.txt

@@ -14,3 +14,4 @@ python-systemd
 pycrypto
 botocore
 boto3
+pytest