Cleanup and enhancements for ansible-test. (#37142)

* Fix type hint typos.
* Add one-time cloud env setup after delegation.
* Add generate_password to util.
* Add username/password support to HttpClient.
* Avoid pip requirement for ansible-test shell.
* Support provisioning Tower instances.

test/runner/lib/cloud/__init__.py

@@ -164,6 +164,7 @@ class CloudBase(ABC):
     _CONFIG_PATH = 'config_path'
     _RESOURCE_PREFIX = 'resource_prefix'
     _MANAGED = 'managed'
+    _SETUP_EXECUTED = 'setup_executed'
 
     def __init__(self, args):
         """
@@ -172,6 +173,20 @@ class CloudBase(ABC):
         self.args = args
         self.platform = self.__module__.split('.')[2]
 
+    @property
+    def setup_executed(self):
+        """
+        :rtype: bool
+        """
+        return self._get_cloud_config(self._SETUP_EXECUTED, False)
+
+    @setup_executed.setter
+    def setup_executed(self, value):
+        """
+        :type value: bool
+        """
+        self._set_cloud_config(self._SETUP_EXECUTED, value)
+
     @property
     def config_path(self):
         """
@@ -214,11 +229,15 @@ class CloudBase(ABC):
         """
         self._set_cloud_config(self._MANAGED, value)
 
-    def _get_cloud_config(self, key):
+    def _get_cloud_config(self, key, default=None):
         """
         :type key: str
+        :type default: str | int | bool | None
         :rtype: str | int | bool
         """
+        if default is not None:
+            return self.args.metadata.cloud_config[self.platform].get(key, default)
+
         return self.args.metadata.cloud_config[self.platform][key]
 
     def _set_cloud_config(self, key, value):
@@ -357,9 +376,21 @@ class CloudProvider(CloudBase):
 
 class CloudEnvironment(CloudBase):
     """Base class for cloud environment plugins. Updates integration test environment after delegation."""
+    def setup_once(self):
+        """Run setup if it has not already been run."""
+        if self.setup_executed:
+            return
+
+        self.setup()
+        self.setup_executed = True
+
+    def setup(self):
+        """Setup which should be done once per environment instead of once per test target."""
+        pass
+
     @abc.abstractmethod
     def configure_environment(self, env, cmd):
-        """
+        """Configuration which should be done once for each test target.
         :type env: dict[str, str]
         :type cmd: list[str]
         """

test/runner/lib/cloud/azure.py

@@ -171,8 +171,8 @@ class AzureCloudEnvironment(CloudEnvironment):
 
 def get_config(config_path):
     """
-    :param config_path: str
+    :type config_path: str
-    :return: dict[str, str]
+    :rtype: dict[str, str]
     """
     with open(config_path, 'r') as config_fd:
         lines = [line for line in config_fd.read().splitlines() if ':' in line and line.strip() and not line.strip().startswith('#')]

test/runner/lib/core_ci.py

@@ -69,6 +69,7 @@ class AnsibleCoreCI(object):
                 'vyos',
                 'junos',
                 'ios',
+                'tower',
             ),
             azure=(
                 'azure',

test/runner/lib/executor.py

@@ -140,6 +140,9 @@ def install_command_requirements(args):
     if not args.requirements:
         return
 
+    if isinstance(args, ShellConfig):
+        return
+
     packages = []
 
     if isinstance(args, TestConfig):
@@ -720,6 +723,9 @@ def command_integration_filtered(args, targets, all_targets):
                 tries -= 1
 
                 try:
+                    if cloud_environment:
+                        cloud_environment.setup_once()
+
                     run_setup_targets(args, test_dir, target.setup_once, all_targets_dict, setup_targets_executed, False)
 
                     start_time = time.time()
@@ -808,12 +814,12 @@ def command_integration_filtered(args, targets, all_targets):
 
 def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, always):
     """
-    :param args: IntegrationConfig
+    :type args: IntegrationConfig
-    :param test_dir: str
+    :type test_dir: str
-    :param target_names: list[str]
+    :type target_names: list[str]
-    :param targets_dict: dict[str, IntegrationTarget]
+    :type targets_dict: dict[str, IntegrationTarget]
-    :param targets_executed: set[str]
+    :type targets_executed: set[str]
-    :param always: bool
+    :type always: bool
     """
     for target_name in target_names:
         if not always and target_name in targets_executed:

test/runner/lib/http.py

@@ -42,6 +42,9 @@ class HttpClient(object):
         self.always = always
         self.insecure = insecure
 
+        self.username = None
+        self.password = None
+
     def get(self, url):
         """
         :type url: str
@@ -83,6 +86,13 @@ class HttpClient(object):
 
         headers['Expect'] = ''  # don't send expect continue header
 
+        if self.username:
+            if self.password:
+                display.sensitive.add(self.password)
+                cmd += ['-u', '%s:%s' % (self.username, self.password)]
+            else:
+                cmd += ['-u', self.username]
+
         for header in headers.keys():
             cmd += ['-H', '%s: %s' % (header, headers[header])]
 

test/runner/lib/manage_ci.py

@@ -135,6 +135,8 @@ class ManagePosixCI(object):
             self.become = ['sudo', '-in', 'PATH=/usr/local/bin:$PATH']
         elif self.core_ci.platform == 'rhel':
             self.become = ['sudo', '-in', 'bash', '-c']
+        elif self.core_ci.platform == 'tower':
+            self.become = ['sudo', '-in', 'bash', '-c']
 
     def setup(self):
         """Start instance and wait for it to become ready and respond to an ansible ping."""

test/runner/lib/sanity/pylint.py

@@ -231,9 +231,9 @@ class PylintTest(SanitySingleVersion):
     def pylint(self, args, context, paths):
         """
         :type args: SanityConfig
-        :param context: str
+        :type context: str
-        :param paths: list[str]
+        :type paths: list[str]
-        :return: list[dict[str, str]]
+        :rtype: list[dict[str, str]]
         """
         rcfile = 'test/sanity/pylint/config/%s' % context
 

test/runner/lib/util.py

@@ -466,6 +466,25 @@ def is_binary_file(path):
         return b'\0' in path_fd.read(1024)
 
 
+def generate_password():
+    """Generate a random password.
+    :rtype: str
+    """
+    chars = [
+        string.ascii_letters,
+        string.digits,
+        string.ascii_letters,
+        string.digits,
+        '-',
+    ] * 4
+
+    password = ''.join([random.choice(char) for char in chars[:-1]])
+
+    display.sensitive.add(password)
+
+    return password
+
+
 class Display(object):
     """Manages color console output."""
     clear = '\033[0m'