Fix ansible-test default image. (#31966)

* Add openssh-client to default docker container.
* Include Azure requirements in default container.

To do so, handling of pip requirements was updated to install each
set of requirements separately and then run a verification pass to
make sure there are no conflicts between requirements.

* Add missing --docker-no-pull option.
* Add documentation for the azure-requirements test.

docs/docsite/rst/dev_guide/testing/sanity/azure-requirements.rst

@@ -0,0 +1,10 @@
+Sanity Tests » azure-requirements
+=================================
+
+Update the Azure integration test requirements file when changes are made to the Azure packaging requirements file:
+
+.. code-block:: bash
+
+    cp packaging/requirements/requirements-azure.txt test/runner/requirements/integration.cloud.azure.txt
+
+This copy of the requirements file is used when building the ``ansible/ansible:default`` Docker container from ``test/runner/Dockerfile``.

test/integration/targets/setup_azure/tasks/main.yml

@@ -1,2 +0,0 @@
-- pip:
-    requirements: '{{ role_path }}/../../../../packaging/requirements/requirements-azure.txt'

test/runner/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get update -y && \
     libxslt1-dev \
     locales \
     make \
+    openssh-client \
     python2.6-dev \
     python2.7-dev \
     python3.5-dev \

test/runner/docker/requirements.sh

@@ -11,13 +11,60 @@ requirements=()
 
 for requirement in *.txt; do
     if [ "${requirement}" != "constraints.txt" ]; then
-        requirements+=("-r" "${requirement}")
+        requirements+=("${requirement}")
     fi
 done
 
 for python_version in "${python_versions[@]}"; do
+    version_requirements=()
+
+    for requirement in "${requirements[@]}"; do
+        case "${python_version}" in
+            "2.6")
+                case "${requirement}" in
+                    "integration.cloud.azure.txt") continue ;;
+                esac
+        esac
+
+        version_requirements+=("${requirement}")
+    done
+
+    echo "==> Installing pip for python ${python_version} ..."
+
     set -x
     "python${python_version}" /tmp/get-pip.py -c constraints.txt
-    "pip${python_version}" install --disable-pip-version-check -c constraints.txt "${requirements[@]}"
     set +x
+
+    echo "==> Installing requirements for python ${python_version} ..."
+
+    for requirement in "${version_requirements[@]}"; do
+        set -x
+        "pip${python_version}" install --disable-pip-version-check -c constraints.txt -r "${requirement}"
+        set +x
+    done
+
+    echo "==> Checking for requirements conflicts for ${python_version} ..."
+
+    after=$("pip${python_version}" list)
+
+    for requirement in "${version_requirements[@]}"; do
+        before="${after}"
+
+        set -x
+        "pip${python_version}" install --disable-pip-version-check -c constraints.txt -r "${requirement}"
+        set +x
+
+        after=$("pip${python_version}" list)
+
+        if [ "${before}" != "${after}" ]; then
+            echo "==> Conflicts detected in requirements for python ${python_version}: ${requirement}"
+            echo ">>> Before"
+            echo "${before}"
+            echo ">>> After"
+            echo "${after}"
+            exit 1
+        fi
+    done
+
+    echo "==> Finished with requirements for python ${python_version}."
 done

test/runner/lib/executor.py

@@ -13,6 +13,7 @@ import textwrap
 import functools
 import shutil
 import stat
+import pipes
 import random
 import string
 import atexit
@@ -162,30 +163,78 @@ def install_command_requirements(args):
         if args.junit:
             packages.append('junit-xml')
 
-    extras = []
+    commands = [generate_pip_install(args.command, packages=packages)]
 
     if isinstance(args, IntegrationConfig):
-        extras += ['cloud.%s' % cp for cp in get_cloud_platforms(args)]
+        for cloud_platform in get_cloud_platforms(args):
+            commands.append(generate_pip_install('%s.cloud.%s' % (args.command, cloud_platform)))
 
-    cmd = generate_pip_install(args.command, packages, extras)
+    # only look for changes when more than one requirements file is needed
+    detect_pip_changes = len(commands) > 1
 
-    if not cmd:
-        return
+    # first pass to install requirements, changes expected unless environment is already set up
+    changes = run_pip_commands(args, commands, detect_pip_changes)
 
-    try:
-        run_command(args, cmd)
-    except SubprocessError as ex:
-        if ex.status != 2:
-            raise
+    if not changes:
+        return  # no changes means we can stop early
 
-        # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.
+    # second pass to check for conflicts in requirements, changes are not expected here
+    changes = run_pip_commands(args, commands, detect_pip_changes)
 
-        # Installing "coverage" on ubuntu 16.04 fails with the error:
-        # AttributeError: 'Requirement' object has no attribute 'project_name'
-        # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
-        # Upgrading pip works around the issue.
-        run_command(args, ['pip', 'install', '--upgrade', 'pip'])
-        run_command(args, cmd)
+    if not changes:
+        return  # no changes means no conflicts
+
+    raise ApplicationError('Conflicts detected in requirements. The following commands reported changes during verification:\n%s' %
+                           '\n'.join((' '.join(pipes.quote(c) for c in cmd) for cmd in changes)))
+
+
+def run_pip_commands(args, commands, detect_pip_changes=False):
+    """
+    :type args: EnvironmentConfig
+    :type commands: list[list[str]]
+    :type detect_pip_changes: bool
+    :rtype: list[list[str]]
+    """
+    changes = []
+
+    after_list = pip_list(args) if detect_pip_changes else None
+
+    for cmd in commands:
+        if not cmd:
+            continue
+
+        before_list = after_list
+
+        try:
+            run_command(args, cmd)
+        except SubprocessError as ex:
+            if ex.status != 2:
+                raise
+
+            # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.
+
+            # Installing "coverage" on ubuntu 16.04 fails with the error:
+            # AttributeError: 'Requirement' object has no attribute 'project_name'
+            # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
+            # Upgrading pip works around the issue.
+            run_command(args, ['pip', 'install', '--upgrade', 'pip'])
+            run_command(args, cmd)
+
+        after_list = pip_list(args) if detect_pip_changes else None
+
+        if before_list != after_list:
+            changes.append(cmd)
+
+    return changes
+
+
+def pip_list(args):
+    """
+    :type args: EnvironmentConfig
+    :rtype: str
+    """
+    stdout, _ = run_command(args, ['pip', 'list'], capture=True, always=True)
+    return stdout
 
 
 def generate_egg_info(args):

test/runner/requirements/integration.cloud.azure.txt

@@ -0,0 +1,16 @@
+packaging
+requests[security]
+azure-mgmt-compute>=2.0.0,<3
+azure-mgmt-network>=1.3.0,<2
+azure-mgmt-storage>=1.2.0,<2
+azure-mgmt-resource>=1.1.0,<2
+azure-storage>=0.35.1,<0.36
+azure-cli-core>=2.0.12,<3
+msrest!=0.4.15
+msrestazure>=0.4.11,<0.5
+azure-mgmt-dns>=1.0.1,<2
+azure-mgmt-keyvault>=0.40.0,<0.41
+azure-mgmt-batch>=4.1.0,<5
+azure-mgmt-sql>=0.7.1,<0.8
+azure-mgmt-web>=0.32.0,<0.33
+azure-mgmt-containerservice>=1.0.0

test/runner/test.py

@@ -255,6 +255,8 @@ def parse_args():
                                      targets=walk_network_integration_targets,
                                      config=NetworkIntegrationConfig)
 
+    add_extra_docker_options(network_integration, integration=False)
+
     network_integration.add_argument('--platform',
                                      metavar='PLATFORM',
                                      action='append',
@@ -272,6 +274,8 @@ def parse_args():
                                      targets=walk_windows_integration_targets,
                                      config=WindowsIntegrationConfig)
 
+    add_extra_docker_options(windows_integration, integration=False)
+
     windows_integration.add_argument('--windows',
                                      metavar='VERSION',
                                      action='append',

test/sanity/code-smell/azure-requirements.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+"""Make sure the Azure requirements files match."""
+
+import filecmp
+
+src = 'packaging/requirements/requirements-azure.txt'
+dst = 'test/runner/requirements/integration.cloud.azure.txt'
+
+if not filecmp.cmp(src, dst):
+    print('Update the Azure integration test requirements with the packaging test requirements:')
+    print('cp %s %s' % (src, dst))
+    exit(1)

test/sanity/code-smell/test-constraints.sh

@@ -5,6 +5,7 @@ constraints=$(
         | grep -v '(sanity_ok)$' \
         | sed 's/ *;.*$//; s/ #.*$//' \
         | grep -v '/constraints.txt:' \
+        | grep -v '/integration.cloud.azure.txt:' \
         | grep '[<>=]'
 )