Add Fedora Python 3 Docker images and fix tests. (#27794)

* Add new Fedora docker images with Python 3.
* Use consistent env var for lookup test.
* Fix testing of virtualenv with Python 3.
* Fix docker_secret tests on Fedora 26.
* Add Python 3 support to Fedora postgresql test.
* Add Python 3 support to Fedora mysql tests.
* Fix uri test server for Python 3 on Fedora.
* Fix iso_extract test for Python 3 on Fedora.
* Add Python 3 support for Fedora to openssl tests.
* Fix dnf group test for Python 3 on Fedora.
* Use force with user deletion in become test.

test/integration/targets/become/tasks/main.yml

@@ -80,3 +80,4 @@
     name: "{{ become_test_user }}"
     state: "absent"
     remove: "yes"
+    force: "yes"

test/integration/targets/dnf/tasks/dnf.yml

@@ -242,9 +242,11 @@
   register: dnf_result
 
 # GROUP INSTALL
-- name: install RPM Development Tools group
+# Using 'Books and Guides' because it is only 5 packages and a 7.3 M download on Fedora 26.
+# It also doesn't install anything that will tamper with our Python environment.
+- name: install Books and Guides group
   dnf:
-    name: "@RPM Development Tools"
+    name: "@Books and Guides"
     state: present
   register: dnf_result
 
@@ -263,14 +265,14 @@
         - "'results' in dnf_result"
 
 # cleanup until https://github.com/ansible/ansible/issues/27377 is resolved
-- shell: dnf -y group install "RPM Development Tools" && dnf -y group remove "RPM Development Tools"
+- shell: dnf -y group install "Books and Guides" && dnf -y group remove "Books and Guides"
 
 # GROUP UPGRADE - this will go to the same method as group install
 # but through group_update - it is its invocation we're testing here
 # see commit 119c9e5d6eb572c4a4800fbe8136095f9063c37b
-- name: install latest RPM Development Tools
+- name: install latest Books and Guides
   dnf:
-    name: "@RPM Development Tools"
+    name: "@Books and Guides"
     state: latest
   register: dnf_result
 
@@ -289,4 +291,4 @@
         - "'results' in dnf_result"
 
 # cleanup until https://github.com/ansible/ansible/issues/27377 is resolved
-- shell: dnf -y group install "RPM Development Tools" && dnf -y group remove "RPM Development Tools"
+- shell: dnf -y group install "Books and Guides" && dnf -y group remove "Books and Guides"

test/integration/targets/docker_secret/tasks/Fedora.yml

@@ -9,9 +9,10 @@
   command: dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
 
 - name: Update cache
-  command: dnf makecache fast 
+  command: dnf makecache
 
 - name: Install docker
   dnf:
     name: docker-ce
     state: present
+    enablerepo: docker-ce-test

test/integration/targets/iso_extract/tasks/7zip.yml

@@ -29,13 +29,20 @@
     state: present
   when: ansible_distribution in ['CentOS']
 
-- name: Install 7zip package if we are on Fedora or CentOS
+- name: Install 7zip package if we are on Fedora
+  dnf:
+    name: p7zip-plugins
+    state: installed
+  become: yes
+  when: ansible_distribution in ['Fedora']
+
+- name: Install 7zip package if we are on CentOS
   yum:
     name: p7zip-plugins
     state: installed
     update_cache: yes
   become: yes
-  when: ansible_distribution in ['Fedora', 'CentOS']
+  when: ansible_distribution in ['CentOS']
 
 - name: Install 7zip package if we are on OpenSUSE
   zypper:

test/integration/targets/lookups/tasks/main.yml

@@ -149,26 +149,21 @@
 
 # ENV LOOKUP
 
-- name: get first environment var name
+- name: get HOME environment var value
-  shell: env | fgrep -v '.' | head -n1 | cut -d\= -f1
+  shell: "echo $HOME"
-  register: known_var_name
+  register: home_var_value
 
-- name: get first environment var value
+- name: use env lookup to get HOME var
-  shell: echo {{ '$' + known_var_name.stdout }}
-  register: known_var_value
-
-- name: use env lookup to get known var
   set_fact:
-    test_val: "{{ lookup('env', known_var_name.stdout) }}"
+    test_val: "{{ lookup('env', 'HOME') }}"
 
-- debug: var=known_var_name.stdout
+- debug: var=home_var_value.stdout
-- debug: var=known_var_value.stdout
 - debug: var=test_val
 
 - name: compare values
   assert:
     that:
-        - "test_val == known_var_value.stdout"
+        - "test_val == home_var_value.stdout"
 
 
 # PIPE LOOKUP

test/integration/targets/setup_mysql_db/vars/Fedora-py3.yml

@@ -0,0 +1,6 @@
+mysql_service: 'mariadb'
+
+mysql_packages:
+    - mariadb-server
+    - python3-mysql
+    - bzip2

test/integration/targets/setup_openssl/vars/RedHat.yml

@@ -1 +1,2 @@
 pyopenssl_package_name: pyOpenSSL
+pyopenssl_package_name_python3: python3-pyOpenSSL

test/integration/targets/setup_postgresql_db/vars/RedHat-py3.yml

@@ -0,0 +1,8 @@
+postgresql_packages:
+  - "postgresql-server"
+  - "python3-psycopg2"
+  - "bzip2"
+  - "xz"
+
+pg_hba_location: "/var/lib/pgsql/data/pg_hba.conf"
+pg_dir: "/var/lib/pgsql/data"

test/integration/targets/uri/files/testserver.py

@@ -5,7 +5,11 @@ if __name__ == '__main__':
         import http.server
         import socketserver
         PORT = int(sys.argv[1])
-        Handler = http.server.SimpleHTTPRequestHandler
+
+        class Handler(http.server.SimpleHTTPRequestHandler):
+            pass
+
+        Handler.extensions_map['.json'] = 'application/json'
         httpd = socketserver.TCPServer(("", PORT), Handler)
         httpd.serve_forever()
     else:

test/runner/setup/docker.sh

@@ -9,6 +9,9 @@ fi
 if [ ! -f /usr/bin/pip ] && [ -f /usr/bin/pip3 ]; then
     ln -s /usr/bin/pip3 /usr/bin/pip
 fi
+if [ ! -f /usr/bin/virtualenv ] && [ -f /usr/bin/virtualenv-3 ]; then
+    ln -s /usr/bin/virtualenv-3 /usr/bin/virtualenv
+fi
 
 # Improve prompts on remote host for interactive use.
 cat << EOF > ~/.bashrc

test/utils/docker/fedora25py3/Dockerfile

@@ -0,0 +1,69 @@
+FROM fedora:25
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*; \
+rm -f /etc/systemd/system/*.wants/*; \
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*; \
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+RUN dnf clean all && \
+    dnf -y --setopt=install_weak_deps=false install \
+    acl \
+    bzip2 \
+    file \
+    findutils \
+    gcc \
+    git \
+    glibc-locale-source \
+    iproute \
+    libffi \
+    libffi-devel \
+    make \
+    mariadb-server \
+    openssh-clients \
+    openssh-server \
+    openssl-devel \
+    procps \
+    python3-cryptography \
+    python3-dbus \
+    python3-devel \
+    python3-dnf \
+    python3-httplib2 \
+    python3-jinja2 \
+    python3-lxml \
+    python3-mock \
+    python3-mysql \
+    python3-nose \
+    python3-paramiko \
+    python3-passlib \
+    python3-pip \
+    python3-PyYAML \
+    python3-setuptools \
+    python3-virtualenv \
+    rubygems \
+    sshpass \
+    subversion \
+    sudo \
+    tar \
+    unzip \
+    which \
+    zip \
+    && \
+    dnf clean all
+
+RUN localedef --quiet -c -i en_US -f UTF-8 en_US.UTF-8
+RUN /usr/bin/sed -i -e 's/^\(Defaults\s*requiretty\)/#--- \1/'  /etc/sudoers
+RUN mkdir /etc/ansible/
+RUN /usr/bin/echo -e '[local]\nlocalhost ansible_connection=local' > /etc/ansible/hosts
+VOLUME /sys/fs/cgroup /run /tmp
+RUN ssh-keygen -q -t dsa -N '' -f /etc/ssh/ssh_host_dsa_key && \
+    ssh-keygen -q -t rsa -N '' -f /etc/ssh/ssh_host_rsa_key && \
+    ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \
+    cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
+    for key in /etc/ssh/ssh_host_*_key.pub; do echo "localhost $(cat ${key})" >> /root/.ssh/known_hosts; done
+RUN pip3 install coverage junit-xml
+ENV container=docker
+CMD ["/usr/sbin/init"]

test/utils/docker/fedora26py3/Dockerfile

@@ -0,0 +1,70 @@
+FROM fedora:26
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*; \
+rm -f /etc/systemd/system/*.wants/*; \
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*; \
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+RUN dnf clean all && \
+    dnf -y --setopt=install_weak_deps=false install \
+    acl \
+    bzip2 \
+    file \
+    findutils \
+    gcc \
+    git \
+    glibc-locale-source \
+    iproute \
+    libffi \
+    libffi-devel \
+    make \
+    mariadb-server \
+    openssh-clients \
+    openssh-server \
+    openssl-devel \
+    procps \
+    python3-cryptography \
+    python3-dbus \
+    python3-devel \
+    python3-dnf \
+    python3-httplib2 \
+    python3-jinja2 \
+    python3-lxml \
+    python3-mock \
+    python3-mysql \
+    python3-nose \
+    python3-paramiko \
+    python3-passlib \
+    python3-pip \
+    python3-PyYAML \
+    python3-setuptools \
+    python3-virtualenv \
+    rpm-build \
+    rubygems \
+    sshpass \
+    subversion \
+    sudo \
+    tar \
+    unzip \
+    which \
+    zip \
+    && \
+    dnf clean all
+
+RUN localedef --quiet -c -i en_US -f UTF-8 en_US.UTF-8
+RUN /usr/bin/sed -i -e 's/^\(Defaults\s*requiretty\)/#--- \1/'  /etc/sudoers
+RUN mkdir /etc/ansible/
+RUN /usr/bin/echo -e '[local]\nlocalhost ansible_connection=local' > /etc/ansible/hosts
+VOLUME /sys/fs/cgroup /run /tmp
+RUN ssh-keygen -q -t dsa -N '' -f /etc/ssh/ssh_host_dsa_key && \
+    ssh-keygen -q -t rsa -N '' -f /etc/ssh/ssh_host_rsa_key && \
+    ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \
+    cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
+    for key in /etc/ssh/ssh_host_*_key.pub; do echo "localhost $(cat ${key})" >> /root/.ssh/known_hosts; done
+RUN pip3 install coverage junit-xml
+ENV container=docker
+CMD ["/usr/sbin/init"]