Run vyos_command Network tests in Shippable (#24514)

* Run VyOS Network tests in Shippable

* net_command no longer exists

* network/ci

* Disable other network tests

* pep8 fix

* Fix PEP 8 issue.

shippable.yml

@@ -24,6 +24,8 @@ matrix:
     - env: TEST=windows/2
     - env: TEST=windows/3
 
+    - env: TEST=network
+
     - env: TEST=linux/centos6/1
     - env: TEST=linux/centos7/1
     - env: TEST=linux/fedora24/1

test/integration/group_vars/vyos.yaml

@@ -1,6 +1,6 @@
 ---
 cli:
   host: "{{ ansible_ssh_host }}"
-  username: "{{ vyos_cli_user | default('ansible-admin') }}"
+#  username: "{{ vyos_cli_user | default('ansible-admin') }}"
-  password: "{{ vyos_cli_pass | default('adminpw') }}"
+#  password: "{{ vyos_cli_pass | default('adminpw') }}"
   transport: cli

test/integration/targets/ios_command/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/ios_config/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/ios_facts/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/ios_system/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/ios_template/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/iosxr_command/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/iosxr_config/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/iosxr_facts/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/iosxr_system/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/iosxr_template/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/net_command/aliases

@@ -1,2 +0,0 @@
-network/basics
-network/ci

test/integration/targets/net_command/tasks/ios.yml

@@ -1,144 +0,0 @@
----
-# Test code for the net_command module.
-# (c) 2017, Red Hat
-
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-
-
-#########################################
-# Positive: Basic Golden path
-#
-- name: net_command - Valid command
-  net_command: show version
-  connection: network_cli
-  register: result
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Ensure output is valid
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|success
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 0
-      - result.stderr == ""
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - "'Cisco Systems' in result.stdout"
-      - "'Cisco' in result.stdout_lines[0]"
-
-
-##########################################
-## Positive: stdout and stderr with rc:0
-##
-#- name: net_command - stdout and stderr, rc:0
-#  net_command: python -c 'import os; os.write(2, "Oh no, this is an error\n"); os.write(1, "This is stdout\n") '
-#  connection: network_cli
-#  register: result
-#
-#- name: Display result
-#  debug:
-#    msg: "{{ result }}"
-#
-#- name: Ensure output is valid
-#  assert:
-#    that:
-#      # net_command should always return "changed", it isn't idempotent
-#      - result|changed
-#      - result|success
-#      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-#      - result.rc == 0
-#      - result.stderr == ""
-#      # Check for platform specific string in output
-#      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-#      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-#      - "'This is stdout' in result.stdout"
-#      - "'Oh no, this is an error' in result.stderr"
-#      - "'Version' in result.stdout_lines[0]"
-#  # FIXME BUG https://github.com/ansible/ansible/issues/20233
-#  # FIXME Once working change test to check for exact return, rather than "in"
-#  ignore_errors: true
-
-#########################################
-# Negative: None zero exit value and stderr
-
-- name: net_command - None zero exit value and stderr
-  net_command: show foobar
-  connection: network_cli
-  register: result
-  ignore_errors: true
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Check we failed
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|failed
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 1
-      - "'Invalid input detected at' in result.stderr"
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.stdout == ""
-      # FIXME BUG: https://github.com/ansible/ansible/issues/20296
-      #- "'Version' in result.stdout_lines[0]"
-
-#########################################
-
-- name: net_command - Missing command
-  net_command:
-  connection: network_cli
-  register: result
-  ignore_errors: true
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Check we failed
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|failed
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 1
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.stdout == ""
-      # FIXME BUG: https://github.com/ansible/ansible/issues/20294
-      - "'Invalid command:' in result.stderr"
-  ignore_errors: true
-
-# Quoting and escape tests - look at command/shell test cases
-#
-#
-# Blocked Test Cases
-#   * Run a command that returns stdout and stderr
-#
-# Run through ansible-lint

test/integration/targets/net_command/tasks/junos.yml

@@ -1,144 +0,0 @@
----
-# Test code for the net_command module.
-# (c) 2017, Red Hat
-
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-
-
-#########################################
-# Positive: Basic Golden path
-#
-- name: net_command - Valid command
-  net_command: show version
-  connection: network_cli
-  register: result
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Ensure output is valid
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|success
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 0
-      - result.stderr == ""
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - "'JUNOS Software Release' in result.stdout"
-      - "'Hostname' in result.stdout_lines[0]"
-
-
-##########################################
-## Positive: stdout and stderr with rc:0
-##
-#- name: net_command - stdout and stderr, rc:0
-#  net_command: python -c 'import os; os.write(2, "Oh no, this is an error\n"); os.write(1, "This is stdout\n") '
-#  connection: network_cli
-#  register: result
-#
-#- name: Display result
-#  debug:
-#    msg: "{{ result }}"
-#
-#- name: Ensure output is valid
-#  assert:
-#    that:
-#      # net_command should always return "changed", it isn't idempotent
-#      - result|changed
-#      - result|success
-#      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-#      - result.rc == 0
-#      - result.stderr == ""
-#      # Check for platform specific string in output
-#      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-#      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-#      - "'This is stdout' in result.stdout"
-#      - "'Oh no, this is an error' in result.stderr"
-#      - "'Version' in result.stdout_lines[0]"
-#  # FIXME BUG https://github.com/ansible/ansible/issues/20233
-#  # FIXME Once working change test to check for exact return, rather than "in"
-#  ignore_errors: true
-
-#########################################
-# Negative: None zero exit value and stderr
-
-- name: net_command - None zero exit value and stderr
-  net_command: show foobar
-  connection: network_cli
-  register: result
-  ignore_errors: true
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Check we failed
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|failed
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 1
-      - "'syntax error, expecting <command>.' in result.stderr"
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.stdout == ""
-      # FIXME BUG: https://github.com/ansible/ansible/issues/20296
-      #- "'Version' in result.stdout_lines[0]"
-
-#########################################
-
-- name: net_command - Missing command
-  net_command:
-  connection: network_cli
-  register: result
-  ignore_errors: true
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Check we failed
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|failed
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 1
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.stdout == ""
-      # FIXME BUG: https://github.com/ansible/ansible/issues/20294
-      - "'Invalid command:' in result.stderr"
-  ignore_errors: true
-
-# Quoting and escape tests - look at command/shell test cases
-#
-#
-# Blocked Test Cases
-#   * Run a command that returns stdout and stderr
-#
-# Run through ansible-lint

test/integration/targets/net_command/tasks/main.yml

@@ -1,46 +0,0 @@
----
-# Test code for the net_command module.
-# (c) 2017, Red Hat
-
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-
-# This is a placeholder test, will be fleshed out by gundalow at a later point
-
-
-# Check if  ansible_network_os being undef or incorrect
-# Without this we wouldn't run any tests, and not actually test Ansible
-
-# FIXME One we can get ansible_network_os from env remove it from 
-# test/runner/johnb@x260:~/git/ansible-inc/ansible/test/runner/lib/executor.py
-#
-
-- name: Check for acceptable ansible_network_os
-  fail:
-      msg: "ansible_network_os is undefined or not an acceptable platform: '{{ ansible_network_os }}'"
-  when: ansible_network_os not in [ 'vyos', 'junos', 'ios' ]
-
-- name: vyos Specific includes
-  include: vyos.yml
-  when: ansible_network_os == 'vyos'
-
-- name: junos Specific includes
-  include: junos.yml
-  when: ansible_network_os == 'junos'
-
-- name: ios Specific includes
-  include: ios.yml
-  when: ansible_network_os == 'ios'

test/integration/targets/net_command/tasks/vyos.yml

@@ -1,144 +0,0 @@
----
-# Test code for the net_command module.
-# (c) 2017, Red Hat
-
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-
-
-#########################################
-# Positive: Basic Golden path
-#
-- name: net_command - Valid command
-  net_command: show version
-  connection: network_cli
-  register: result
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Ensure output is valid
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|success
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 0
-      - result.stderr == ""
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - "'VyOS' in result.stdout"
-      - "'Version' in result.stdout_lines[0]"
-
-
-#########################################
-# Positive: stdout and stderr with rc:0
-#
-- name: net_command - stdout and stderr, rc:0
-  net_command: python -c 'import os; os.write(2, "Oh no, this is an error\n"); os.write(1, "This is stdout\n") '
-  connection: network_cli
-  register: result
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Ensure output is valid
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|success
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 0
-      - result.stderr == ""
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - "'This is stdout' in result.stdout"
-      - "'Oh no, this is an error' in result.stderr"
-      - "'Version' in result.stdout_lines[0]"
-  # FIXME BUG https://github.com/ansible/ansible/issues/20233
-  # FIXME Once working change test to check for exact return, rather than "in"
-  ignore_errors: true
-
-#########################################
-# Negative: None zero exit value and stderr
-
-- name: net_command - None zero exit value and stderr
-  net_command: show foobar
-  connection: network_cli
-  register: result
-  ignore_errors: true
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Check we failed
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|failed
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 1
-      - "'Invalid command:' in result.stderr"
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.stdout == ""
-      # FIXME BUG: https://github.com/ansible/ansible/issues/20296
-      #- "'Version' in result.stdout_lines[0]"
-
-#########################################
-
-- name: net_command - Missing command
-  net_command:
-  connection: network_cli
-  register: result
-  ignore_errors: true
-
-- name: Display result
-  debug:
-    msg: "{{ result }}"
-
-- name: Check we failed
-  assert:
-    that:
-      # net_command should always return "changed", it isn't idempotent
-      - result|changed
-      - result|failed
-      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
-      - result.rc == 1
-      # Check for platform specific string in output
-      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
-      - result.stdout == ""
-      # FIXME BUG: https://github.com/ansible/ansible/issues/20294
-      - "'Invalid command:' in result.stderr"
-  ignore_errors: true
-
-# Quoting and escape tests - look at command/shell test cases
-#
-#
-# Blocked Test Cases
-#   * Run a command that returns stdout and stderr
-#
-# Run through ansible-lint

test/integration/targets/nxos_command/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_config/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_evpn_global/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_facts/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_feature/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_interface/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_mtu/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_nxapi/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_system/aliases

@@ -1 +0,0 @@
-network/ci

test/integration/targets/nxos_template/aliases

@@ -1 +0,0 @@
-network/ci

test/runner/lib/executor.py

@@ -320,6 +320,7 @@ def network_inventory(remotes):
             ansible_user=remote.connection.username,
             ansible_ssh_private_key_file=remote.ssh_key.key,
             ansible_network_os=remote.platform,
+            ansible_connection='local'
         )
 
         groups[remote.platform].append(

test/runner/lib/manage_ci.py

@@ -73,17 +73,23 @@ class ManageNetworkCI(object):
         """Wait for instance to respond to ansible ping."""
         extra_vars = [
             'ansible_host=%s' % self.core_ci.connection.hostname,
-            'ansible_user=%s' % self.core_ci.connection.username,
             'ansible_port=%s' % self.core_ci.connection.port,
             'ansible_connection=local',
             'ansible_ssh_private_key_file=%s' % self.core_ci.ssh_key.key,
-            'ansible_network_os=%s' % self.core_ci.platform,
         ]
 
         name = '%s-%s' % (self.core_ci.platform, self.core_ci.version.replace('.', '_'))
 
         env = ansible_environment(self.core_ci.args)
-        cmd = ['ansible', '-m', 'net_command', '-a', '?', '-i', '%s,' % name, name, '-e', ' '.join(extra_vars)]
+        cmd = [
+            'ansible',
+            '-m', '%s_command' % self.core_ci.platform,
+            '-a', 'commands=?',
+            '-u', self.core_ci.connection.username,
+            '-i', '%s,' % name,
+            '-e', ' '.join(extra_vars),
+            name,
+        ]
 
         for _ in range(1, 90):
             try: