[stable-1] Update CI (#1790)

* Update CI (#1782)

* Update targets for CI for devel branch; move some targets to stable-2.10.

* Skipping test on RHEL 8.3 (it is already skipped on RHEL 8.2).

* Linting.

* Shut 2.9/2.10 pylint complaints up.

* More sanity.

* Bump CI to FreeBSD 11.4, 12.2. (#1657)

* Bump CI to FreeBSD 11.4, 12.2.

* Make FreeBSD Python package selection more future-proof.

(cherry picked from commit c1eb0a232c)

* Add macOS 11.1 tests (#1619)

* Add macOS 11.1 tests.

* Hopefully fix virtualenv.sh problems.

(cherry picked from commit 74174f11ff)

* Skip all postgresql tests on FreeBSD.

* Skip kubevirt inventory tests on macOS.

.azure-pipelines/azure-pipelines.yml

@@ -140,18 +140,16 @@ stages:
         parameters:
           testFormat: devel/{0}
           targets:
-            - name: OS X 10.11
+            - name: macOS 11.1
-              test: osx/10.11
+              test: macos/11.1
-            - name: macOS 10.15
+            - name: RHEL 7.9
-              test: macos/10.15
+              test: rhel/7.9
-            - name: RHEL 7.8
+            - name: RHEL 8.3
-              test: rhel/7.8
+              test: rhel/8.3
-            - name: RHEL 8.2
+            - name: FreeBSD 11.4
-              test: rhel/8.2
+              test: freebsd/11.4
-            - name: FreeBSD 11.1
+            - name: FreeBSD 12.2
-              test: freebsd/11.1
+              test: freebsd/12.2
-            - name: FreeBSD 12.1
-              test: freebsd/12.1
           groups:
             - 1
             - 2
@@ -168,6 +166,12 @@ stages:
           targets:
             - name: OS X 10.11
               test: osx/10.11
+            - name: macOS 10.15
+              test: macos/10.15
+            - name: macOS 11.1
+              test: macos/11.1
+            - name: RHEL 7.8
+              test: rhel/7.8
             - name: RHEL 8.2
               test: rhel/8.2
             - name: FreeBSD 12.1
@@ -220,8 +224,6 @@ stages:
               test: opensuse15py2
             - name: openSUSE 15 py3
               test: opensuse15
-            - name: Ubuntu 16.04
-              test: ubuntu1604
             - name: Ubuntu 18.04
               test: ubuntu1804
           groups:
@@ -244,8 +246,10 @@ stages:
               test: fedora32
             - name: openSUSE 15 py3
               test: opensuse15
-            - name: Ubuntu 18.04
+            - name: Ubuntu 16.04
-              test: ubuntu1804
+              test: ubuntu1604
+            #- name: Ubuntu 18.04
+            #  test: ubuntu1804
           groups:
             - 1
             - 2

plugins/inventory/proxmox.py

@@ -212,10 +212,8 @@ class InventoryModule(BaseInventoryPlugin, Cacheable):
                 if config == 'rootfs' or config.startswith(('virtio', 'sata', 'ide', 'scsi')):
                     value = ('disk_image=' + value)
 
-                if isinstance(value, int) or ',' not in value:
+                if not (isinstance(value, int) or ',' not in value):
-                    value = value
+                    # split off strings with commas to a dict
-                # split off strings with commas to a dict
-                else:
                     # skip over any keys that cannot be processed
                     try:
                         value = dict(key.split("=") for key in value.split(","))

plugins/inventory/virtualbox.py

@@ -217,7 +217,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
                     yield host not in v
             yield True
 
-        return all([found_host for found_host in find_host(host, inventory)])
+        return all(find_host(host, inventory))
 
     def verify_file(self, path):
 

plugins/module_utils/cloud.py

@@ -130,7 +130,7 @@ class CloudRetry(object):
                     try:
                         return f(*args, **kwargs)
                     except Exception as e:
-                        if isinstance(e, cls.base_class):
+                        if isinstance(e, cls.base_class):  # pylint: disable=isinstance-second-argument-not-valid-type
                             response_code = cls.status_code_from_exception(e)
                             if cls.found(response_code, catch_extra_error_codes):
                                 msg = "{0}: Retrying in {1} seconds...".format(str(e), delay)

plugins/module_utils/module_helper.py

@@ -259,7 +259,7 @@ class CmdMixin(object):
 
     def _calculate_args(self, extra_params=None, params=None):
         def add_arg_formatted_param(_cmd_args, arg_format, _value):
-            args = [x for x in arg_format.to_text(_value)]
+            args = list(arg_format.to_text(_value))
             return _cmd_args + args
 
         def find_format(_param):

plugins/modules/cloud/ovirt/ovirt_tag_facts.py

@@ -137,17 +137,13 @@ def main():
             host = search_by_name(hosts_service, module.params['host'])
             if host is None:
                 raise Exception("Host '%s' was not found." % module.params['host'])
-            tags.extend([
+            tags.extend(hosts_service.host_service(host.id).tags_service().list())
-                tag for tag in hosts_service.host_service(host.id).tags_service().list()
-            ])
         if module.params['vm']:
             vms_service = connection.system_service().vms_service()
             vm = search_by_name(vms_service, module.params['vm'])
             if vm is None:
                 raise Exception("Vm '%s' was not found." % module.params['vm'])
-            tags.extend([
+            tags.extend(vms_service.vm_service(vm.id).tags_service().list())
-                tag for tag in vms_service.vm_service(vm.id).tags_service().list()
-            ])
 
         if not (module.params['vm'] or module.params['host'] or module.params['name']):
             tags = all_tags

plugins/modules/net_tools/hetzner_firewall.py

@@ -323,7 +323,7 @@ def normalize_ip(ip, ip_version):
     if '/' in ip:
         ip, range = ip.split('/')
     else:
-        ip, range = ip, ''
+        range = ''
     ip_addr = to_native(compat_ipaddress.ip_address(to_text(ip)).compressed)
     if range == '':
         range = '32' if ip_version.lower() == 'ipv4' else '128'

plugins/modules/packaging/os/openbsd_pkg.py

@@ -164,7 +164,7 @@ def get_package_state(names, pkg_spec, module):
         if stdout:
             # If the requested package name is just a stem, like "python", we may
             # find multiple packages with that name.
-            pkg_spec[name]['installed_names'] = [installed_name for installed_name in stdout.splitlines()]
+            pkg_spec[name]['installed_names'] = stdout.splitlines()
             module.debug("get_package_state(): installed_names = %s" % pkg_spec[name]['installed_names'])
             pkg_spec[name]['installed_state'] = True
         else:

scripts/inventory/freeipa.py

@@ -67,10 +67,10 @@ def list_groups(api):
         members = []
 
         if 'member_host' in hostgroup:
-            members = [host for host in hostgroup['member_host']]
+            members = list(hostgroup['member_host'])
         if 'memberindirect_host' in hostgroup:
             members += (host for host in hostgroup['memberindirect_host'])
-        inventory[hostgroup['cn'][0]] = {'hosts': [host for host in members]}
+        inventory[hostgroup['cn'][0]] = {'hosts': list(members)}
 
         for member in members:
             hostvars[member] = {}

scripts/inventory/gce.py

@@ -326,8 +326,8 @@ class GceInventory(object):
         If provided, this will be used to filter the results of the grouped_instances call'''
         import csv
         reader = csv.reader([os.environ.get('GCE_ZONE', "")], skipinitialspace=True)
-        zones = [r for r in reader]
+        zones = list(reader)
-        return [z for z in zones[0]]
+        return list(zones[0])
 
     def parse_cli_args(self):
         ''' Command line argument processing '''

scripts/inventory/lxc_inventory.py

@@ -38,7 +38,7 @@ def build_dict():
                        for c in lxc.list_containers()])
 
     # Extract the groups, flatten the list, and remove duplicates
-    groups = set(sum([g for g in containers.values()], []))
+    groups = set(sum(containers.values(), []))
 
     # Create a dictionary for each group (including the 'all' group
     return dict([(g, {'hosts': [k for k, v in containers.items() if g in v],

tests/integration/targets/filter_jc/runme.sh

@@ -2,6 +2,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1  # see https://github.com/ansible/ansible/pull/73000#issuecomment-757012395; can be removed once Ansible 2.9 and ansible-base 2.10 support has been dropped
 source virtualenv.sh
 
 # Requirements have to be installed prior to running ansible-playbook

tests/integration/targets/filter_json_query/runme.sh

@@ -2,6 +2,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1  # see https://github.com/ansible/ansible/pull/73000#issuecomment-757012395; can be removed once Ansible 2.9 and ansible-base 2.10 support has been dropped
 source virtualenv.sh
 
 # Requirements have to be installed prior to running ansible-playbook

tests/integration/targets/inventory_kubevirt/aliases

@@ -1 +1,2 @@
 shippable/posix/group2
+skip/macos

tests/integration/targets/postgresql_copy/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_db/aliases

@@ -4,3 +4,4 @@ postgresql_db
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_idx/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_lang/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_membership/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_owner/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_pg_hba/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_ping/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_privs/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_query/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_schema/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_sequence/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_set/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_slot/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_table/aliases

@@ -3,3 +3,4 @@ shippable/posix/group5
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_tablespace/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/postgresql_user/aliases

@@ -3,3 +3,4 @@ shippable/posix/group4
 skip/aix
 skip/osx
 skip/macos
+skip/freebsd

tests/integration/targets/setup_openssl/vars/FreeBSD.yml

@@ -1,3 +1,3 @@
 pyopenssl_package_name: py27-openssl
-pyopenssl_package_name_python3: py36-openssl
+pyopenssl_package_name_python3: "py{{ ansible_python.version.major }}{{ ansible_python.version.minor }}-openssl"
 openssl_package_name: openssl

tests/integration/targets/setup_postgresql_db/vars/FreeBSD-11-py3.yml

@@ -1,6 +1,6 @@
 postgresql_packages:
   - postgresql95-server
-  - py36-psycopg2
+  - "py{{ ansible_python.version.major }}{{ ansible_python.version.minor }}-psycopg2"
 
 pg_dir: /usr/local/pgsql/data
 pg_hba_location: "{{ pg_dir }}/pg_hba.conf"

tests/integration/targets/setup_postgresql_db/vars/FreeBSD-12.0-py3.yml

@@ -1,6 +1,6 @@
 postgresql_packages:
   - postgresql95-server
-  - py36-psycopg2
+  - "py{{ ansible_python.version.major }}{{ ansible_python.version.minor }}-psycopg2"
 
 pg_dir: /usr/local/pgsql/data
 pg_hba_location: "{{ pg_dir }}/pg_hba.conf"

tests/integration/targets/setup_postgresql_db/vars/FreeBSD-12.1-py3.yml

@@ -1,6 +1,6 @@
 postgresql_packages:
   - postgresql11-server
-  - py36-psycopg2
+  - "py{{ ansible_python.version.major }}{{ ansible_python.version.minor }}-psycopg2"
 
 pg_dir: /var/db/postgres/data11
 pg_hba_location: "{{ pg_dir }}/pg_hba.conf"

tests/integration/targets/xml/tasks/main.yml

@@ -6,7 +6,7 @@
 
 - name: Install lxml (FreeBSD)
   package:
-    name: '{{ "py27-lxml" if ansible_python.version.major == 2 else "py36-lxml" }}'
+    name: 'py{{ ansible_python.version.major }}{{ ansible_python.version.minor }}-lxml'
     state: present
   when: ansible_os_family == "FreeBSD"
 

tests/sanity/ignore-2.10.txt

@@ -1,5 +1,6 @@
 plugins/callback/hipchat.py pylint:blacklisted-name
 plugins/connection/lxc.py pylint:blacklisted-name
+plugins/module_utils/cloud.py pylint:bad-option-value  # a pylint test that is disabled was modified over time
 plugins/module_utils/compat/ipaddress.py no-assert
 plugins/module_utils/compat/ipaddress.py no-unicode-literals
 plugins/module_utils/_mount.py future-import-boilerplate
@@ -552,6 +553,7 @@ plugins/modules/system/xfconf.py validate-modules:return-syntax-error
 plugins/modules/web_infrastructure/jenkins_plugin.py use-argspec-type-path
 plugins/modules/web_infrastructure/rundeck_acl_policy.py pylint:blacklisted-name
 scripts/inventory/gce.py pylint:blacklisted-name
+tests/unit/plugins/module_utils/postgresql/test_postgres.py pylint:bad-option-value  # a pylint test that is disabled was modified over time
 tests/unit/plugins/modules/cloud/google/test_gcp_forwarding_rule.py future-import-boilerplate
 tests/unit/plugins/modules/cloud/google/test_gcp_forwarding_rule.py metaclass-boilerplate
 tests/utils/shippable/check_matrix.py replace-urlopen

tests/sanity/ignore-2.11.txt

@@ -14,29 +14,18 @@ plugins/modules/cloud/centurylink/clc_server.py validate-modules:parameter-list-
 plugins/modules/cloud/centurylink/clc_server_snapshot.py validate-modules:parameter-list-no-elements
 plugins/modules/cloud/docker/docker_container.py use-argspec-type-path # uses colon-separated paths, can't use type=path
 plugins/modules/cloud/google/gcdns_record.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce.py validate-modules:parameter-list-no-elements
 plugins/modules/cloud/google/gce.py yamllint:unparsable-with-libyaml
-plugins/modules/cloud/google/gce_eip.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_eip.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_img.py pylint:blacklisted-name
-plugins/modules/cloud/google/gce_instance_template.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_instance_template.py validate-modules:doc-missing-type
 plugins/modules/cloud/google/gce_instance_template.py validate-modules:parameter-list-no-elements
 plugins/modules/cloud/google/gce_labels.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_lb.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_lb.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_mig.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_mig.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_net.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_net.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_pd.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_pd.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_snapshot.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_snapshot.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gce_tag.py pylint:blacklisted-name
 plugins/modules/cloud/google/gce_tag.py validate-modules:parameter-list-no-elements
-plugins/modules/cloud/google/gcp_backend_service.py pylint:blacklisted-name
 plugins/modules/cloud/google/gcp_backend_service.py validate-modules:parameter-list-no-elements
 plugins/modules/cloud/google/gcp_forwarding_rule.py validate-modules:parameter-list-no-elements
 plugins/modules/cloud/google/gcp_healthcheck.py pylint:blacklisted-name
@@ -551,7 +540,6 @@ plugins/modules/system/xfconf.py validate-modules:parameter-state-invalid-choice
 plugins/modules/system/xfconf.py validate-modules:return-syntax-error
 plugins/modules/web_infrastructure/jenkins_plugin.py use-argspec-type-path
 plugins/modules/web_infrastructure/rundeck_acl_policy.py pylint:blacklisted-name
-scripts/inventory/gce.py pylint:blacklisted-name
 tests/unit/plugins/modules/cloud/google/test_gcp_forwarding_rule.py future-import-boilerplate
 tests/unit/plugins/modules/cloud/google/test_gcp_forwarding_rule.py metaclass-boilerplate
 tests/utils/shippable/check_matrix.py replace-urlopen

tests/sanity/ignore-2.9.txt

@@ -1,5 +1,6 @@
 plugins/callback/hipchat.py pylint:blacklisted-name
 plugins/connection/lxc.py pylint:blacklisted-name
+plugins/module_utils/cloud.py pylint:bad-option-value  # a pylint test that is disabled was modified over time
 plugins/module_utils/compat/ipaddress.py no-assert
 plugins/module_utils/compat/ipaddress.py no-unicode-literals
 plugins/module_utils/_mount.py future-import-boilerplate
@@ -482,6 +483,7 @@ plugins/modules/web_infrastructure/nginx_status_facts.py validate-modules:deprec
 plugins/modules/web_infrastructure/nginx_status_facts.py validate-modules:invalid-documentation
 plugins/modules/web_infrastructure/rundeck_acl_policy.py pylint:blacklisted-name
 scripts/inventory/gce.py pylint:blacklisted-name
+tests/unit/plugins/module_utils/postgresql/test_postgres.py pylint:bad-option-value  # a pylint test that is disabled was modified over time
 tests/unit/plugins/modules/cloud/google/test_gcp_forwarding_rule.py future-import-boilerplate
 tests/unit/plugins/modules/cloud/google/test_gcp_forwarding_rule.py metaclass-boilerplate
 tests/utils/shippable/check_matrix.py replace-urlopen

tests/unit/plugins/module_utils/postgresql/test_postgres.py

@@ -225,8 +225,8 @@ class TestConnectToDb():
         db_connection = pg.connect_to_db(m_ansible_module, conn_params)
         cursor = db_connection.cursor()
         # if errors, db_connection returned as None:
-        assert isinstance(db_connection, DbConnection)
+        assert isinstance(db_connection, DbConnection)  # pylint: disable=isinstance-second-argument-not-valid-type
-        assert isinstance(cursor, Cursor)
+        assert isinstance(cursor, Cursor)  # pylint: disable=isinstance-second-argument-not-valid-type
         assert m_ansible_module.err_msg == ''
         # The default behaviour, normal in this case:
         assert 'Database name has not been passed' in m_ansible_module.warn_msg
@@ -241,8 +241,8 @@ class TestConnectToDb():
         db_connection = pg.connect_to_db(m_ansible_module, conn_params)
         cursor = db_connection.cursor()
         # if errors, db_connection returned as None:
-        assert isinstance(db_connection, DbConnection)
+        assert isinstance(db_connection, DbConnection)  # pylint: disable=isinstance-second-argument-not-valid-type
-        assert isinstance(cursor, Cursor)
+        assert isinstance(cursor, Cursor)  # pylint: disable=isinstance-second-argument-not-valid-type
         assert m_ansible_module.err_msg == ''
         # The default behaviour, normal in this case:
         assert 'Database name has not been passed' in m_ansible_module.warn_msg
@@ -292,8 +292,8 @@ class TestConnectToDb():
         cursor = db_connection.cursor()
 
         # if errors, db_connection returned as None:
-        assert isinstance(db_connection, DbConnection)
+        assert isinstance(db_connection, DbConnection)  # pylint: disable=isinstance-second-argument-not-valid-type
-        assert isinstance(cursor, Cursor)
+        assert isinstance(cursor, Cursor)  # pylint: disable=isinstance-second-argument-not-valid-type
         assert m_ansible_module.err_msg == ''
 
         # case 2: psycopg2.__version < 2.4.2
@@ -305,8 +305,8 @@ class TestConnectToDb():
         cursor = db_connection.cursor()
 
         # if errors, db_connection returned as None:
-        assert isinstance(db_connection, DbConnection)
+        assert isinstance(db_connection, DbConnection)  # pylint: disable=isinstance-second-argument-not-valid-type
-        assert isinstance(cursor, Cursor)
+        assert isinstance(cursor, Cursor)  # pylint: disable=isinstance-second-argument-not-valid-type
         assert 'psycopg2 must be at least 2.4.3' in m_ansible_module.err_msg
 
 

tests/unit/plugins/modules/monitoring/test_monit.py

@@ -97,7 +97,7 @@ class MonitTest(unittest.TestCase):
                 self.monit.wait_for_monit_to_stop_pending()
 
 
-@pytest.mark.parametrize('status_name', [name for name in monit.StatusValue.ALL_STATUS])
+@pytest.mark.parametrize('status_name', monit.StatusValue.ALL_STATUS)
 def test_status_value(status_name):
     value = getattr(monit.StatusValue, status_name.upper())
     status = monit.StatusValue(value)

tests/unit/plugins/modules/system/interfaces_file/test_interfaces_file.py

@@ -60,7 +60,7 @@ class TestInterfacesFileModule(unittest.TestCase):
                                              tofile=os.path.basename(backup))
         # Restore backup
         move(backup, path)
-        deltas = [d for d in diffs]
+        deltas = list(diffs)
         self.assertTrue(len(deltas) == 0)
 
     def compareInterfacesLinesToFile(self, interfaces_lines, path, testname=None):