Update nxos_install_os module (#40102)

* Add nxos_install_os integration tests

* Update call to check timers

* Update check_ansible_timer method

* Modify network_cli integration tests

* Add timer check for nxos_install_os

* Add comments for clear_persistent_sockets

* Update connection info for tests

* More updates

* Restructure files for provider and non-provider testing

* Update env var name and add check for ISSU switchover

lib/ansible/modules/network/nxos/nxos_install_os.py

@@ -35,13 +35,9 @@ notes:
       - N9k 7.0(3)I4(6), 7.0(3)I5(3), 7.0(3)I6(1), 7.0(3)I7(1), 7.0(3)F2(2), 7.0(3)F3(2)
       - N3k 6.0(2)A8(6), 6.0(2)A8(8), 7.0(3)I6(1), 7.0(3)I7(1)
       - N7k 7.3(0)D1(1), 8.0(1), 8.2(1)
-    - This module executes longer then the default ansible timeout value and
-      will generate errors unless the module timeout parameter is set to a
-      value of 500 seconds or higher.
-      The example time is sufficent for most upgrades but this can be
-      tuned higher based on specific upgrade time requirements.
-      The module will exit with a failure message if the timer is
-      not set to 500 seconds or higher.
+    - This module requires both the ANSIBLE_PERSISTENT_CONNECT_TIMEOUT and
+      ANSIBLE_PERSISTENT_COMMAND_TIMEOUT timers to be set to 600 seconds or higher.
+      The module will exit if the timers are not set properly.
     - Do not include full file paths, just the name of the file(s) stored on
       the top level flash directory.
     - This module attempts to install the software immediately,
@@ -81,7 +77,6 @@ EXAMPLES = '''
   nxos_install_os:
     system_image_file: nxos.7.0.3.I6.1.bin
     issu: desired
-    provider: "{{ connection | combine({'timeout': 500}) }}"
 
 - name: Wait for device to come back up with new image
   wait_for:
@@ -129,25 +124,6 @@ from ansible.module_utils.network.nxos.nxos import nxos_argument_spec, check_arg
 from ansible.module_utils.basic import AnsibleModule
 
 
-def check_ansible_timer(module):
-    '''Check Ansible Timer Values'''
-    msg = "The 'timeout' provider param value for this module to execute\n"
-    msg = msg + 'properly is too low.\n'
-    msg = msg + 'Upgrades can take a long time so the value needs to be set\n'
-    msg = msg + 'to the recommended value of 500 seconds or higher in the\n'
-    msg = msg + 'ansible playbook for the nxos_install_os module.\n'
-    msg = msg + '\n'
-    msg = msg + 'provider: "{{ connection | combine({\'timeout\': 500}) }}"'
-    data = module.params.get('provider')
-    timer_low = False
-    if data.get('timeout') is None:
-        timer_low = True
-    if data.get('timeout') is not None and data.get('timeout') < 500:
-        timer_low = True
-    if timer_low:
-        module.fail_json(msg=msg.split('\n'))
-
-
 # Output options are 'text' or 'json'
 def execute_show_command(module, command, output='text'):
     cmds = [{
@@ -276,6 +252,9 @@ def parse_show_install(data):
         if re.search(r'Install has been successful', x):
             ud['upgrade_succeeded'] = True
             break
+        if re.search(r'Switching over onto standby', x):
+            ud['upgrade_succeeded'] = True
+            break
 
         # We get these messages when the upgrade is non-disruptive and
         # we loose connection with the switchover but far enough along that
@@ -539,10 +518,6 @@ def main():
     warnings = list()
     check_args(module, warnings)
 
-    # This module will error out if the Ansible task timeout value is not
-    # tuned high enough.
-    check_ansible_timer(module)
-
     # Get system_image_file(sif), kickstart_image_file(kif) and
     # issu settings from module params.
     sif = module.params['system_image_file']

lib/ansible/plugins/action/nxos.py

@@ -55,6 +55,12 @@ class ActionModule(_ActionModule):
             elif self._play_context.connection == 'local':
                 self._task.args['username'] = self._play_context.connection_user
 
+        if self._task.action == 'nxos_install_os':
+            if C.PERSISTENT_COMMAND_TIMEOUT < 600 or C.PERSISTENT_CONNECT_TIMEOUT < 600:
+                msg = 'PERSISTENT_COMMAND_TIMEOUT and PERSISTENT_CONNECT_TIMEOUT'
+                msg += ' must be set to 600 seconds or higher when using nxos_install_os module'
+                return {'failed': True, 'msg': msg}
+
         if self._play_context.connection in ('network_cli', 'httpapi'):
             provider = self._task.args.get('provider', {})
             if any(provider.values()):

test/integration/targets/nxos_install_os/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+testcase: "upgrade"

test/integration/targets/nxos_install_os/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - prepare_nxos_tests

test/integration/targets/nxos_install_os/tasks/httpapi.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect common test cases
+  find:
+    paths: "{{ role_path }}/tests/common"
+    patterns: "{{ testcase }}.yaml"
+  connection: local
+  register: test_cases
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test cases (ansible_connection=httpapi)
+  include: "{{ test_case_to_run }} ansible_connection=httpapi"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/nxos_install_os/tasks/main.yaml

@@ -0,0 +1,5 @@
+---
+- { include: network_cli.yaml, tags: ['cli'] }
+- { include: network_local.yaml, tags: ['local'] }
+- { include: httpapi.yaml, tags: ['httpapi'] }
+- { include: nxapi.yaml, tags: ['nxapi'] }

test/integration/targets/nxos_install_os/tasks/network_cli.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect common test cases
+  find:
+    paths: "{{ role_path }}/tests/common"
+    patterns: "{{ testcase }}.yaml"
+  connection: local
+  register: test_cases
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test cases (ansible_connection=network_cli)
+  include: "{{ test_case_to_run }} ansible_connection=network_cli"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/nxos_install_os/tasks/network_local.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect common test cases
+  find:
+    paths: "{{ role_path }}/tests/common"
+    patterns: "{{ testcase }}.yaml"
+  connection: local
+  register: test_cases
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (ansible_connection=local transport=ssh)
+  include: "{{ test_case_to_run }} ansible_connection=local connection={{ cli }}"
+  with_first_found: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/nxos_install_os/tasks/nxapi.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect common test cases
+  find:
+    paths: "{{ role_path }}/tests/common"
+    patterns: "{{ testcase }}.yaml"
+  connection: local
+  register: test_cases
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test cases (ansible_connection=local transport=nxapi)
+  include: "{{ test_case_to_run }} ansible_connection=local connection={{ nxapi }}"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/nxos_install_os/tasks/upgrade/clear_persistent_sockets.yaml

@@ -0,0 +1,19 @@
+---
+# This playbook is only provided for reference as a brute force way to
+# clear persistent connections on an Ansible server.  This was a workaround
+# for a problem with meta: reset_connection but should not be used in
+# ansible release 2.6 or later.
+- name: Clean up sockets with file module
+  file:
+    state: absent
+    path: "{{ home }}/.ansible/pc/"
+  delegate_to: 127.0.0.1
+
+- name: "Display socket info after delete"
+  shell: "/bin/ls {{ home }}/.ansible"
+  args:
+    executable: /bin/bash
+  delegate_to: 127.0.0.1
+  register: output
+
+- debug: msg="Local Socket Info {{ output['stdout_lines'] }}"

test/integration/targets/nxos_install_os/tasks/upgrade/copy_kick_system_images.yaml

@@ -0,0 +1,41 @@
+---
+- set_fact: ignore_errors_httpapi='no'
+- set_fact: ignore_errors_httpapi='yes'
+  when: ansible_connection == 'httpapi'
+
+- include: targets/nxos_install_os/tasks/upgrade/enable_scp_server.yaml
+  when: connection is not defined
+
+- include: targets/nxos_install_os/tasks/upgrade/enable_scp_server_provider.yaml
+  when: connection is defined
+
+- name: "Copy {{ si }} to bootflash"
+  nxos_file_copy:
+    local_file: "{{image_dir}}{{ si }}"
+    file_system: "bootflash:"
+  register: result
+  ignore_errors: "{{ ignore_errors_httpapi }}"
+
+- name: "Copy {{ si }} to bootflash"
+  nxos_file_copy:
+    local_file: "{{image_dir}}{{ si }}"
+    file_system: "bootflash:"
+  register: result
+  ignore_errors: "{{ ignore_errors_httpapi }}"
+
+- debug:
+    msg: "{{ item.key }} {{ item.value }}"
+  with_dict: "{{ result }}"
+
+- name: "Copy {{ ki }} to bootflash"
+  nxos_file_copy:
+    local_file: "{{image_dir}}/{{ ki }}"
+    file_system: "bootflash:"
+  register: result
+  when: ki is defined
+  ignore_errors: "{{ ignore_errors_httpapi }}"
+
+- debug:
+    msg: "{{ item.key }} {{ item.value }}"
+  with_dict: "{{ result }}"
+  when: ki is defined

test/integration/targets/nxos_install_os/tasks/upgrade/delete_files.yaml

@@ -0,0 +1,10 @@
+---
+- name: "Delete Files To Make Room On Bootflash"
+  nxos_config: &remove_file
+    lines:
+      - terminal dont-ask
+      - allow delete boot-image
+      - "delete {{ item }}"
+    match: none
+  ignore_errors: yes
+  with_items: "{{ delete_image_list }}"

test/integration/targets/nxos_install_os/tasks/upgrade/delete_files_provider.yaml

@@ -0,0 +1,11 @@
+---
+ - name: "Delete Files To Make Room On Bootflash using provider"
+   nxos_config: &remove_file
+     lines:
+       - terminal dont-ask
+       - allow delete boot-image
+       - "delete {{ item }}"
+     match: none
+     provider: "{{ connection }}"
+   ignore_errors: yes
+   with_items: "{{ delete_image_list }}"

test/integration/targets/nxos_install_os/tasks/upgrade/enable_scp_server.yaml

@@ -0,0 +1,5 @@
+---
+- name: "Setup - Turn on feature scp-server"
+  nxos_feature:
+    feature: scp-server
+    state: enabled

test/integration/targets/nxos_install_os/tasks/upgrade/enable_scp_server_provider.yaml

@@ -0,0 +1,6 @@
+---
+- name: "Setup - Turn on feature scp-server using provider"
+  nxos_feature:
+    feature: scp-server
+    state: enabled
+    provider: "{{ connection }}"

test/integration/targets/nxos_install_os/tasks/upgrade/install_os.yaml

@@ -0,0 +1,23 @@
+---
+- include: targets/nxos_install_os/tasks/upgrade/delete_files.yaml
+  when: delete_image_list is defined
+
+- include: targets/nxos_install_os/tasks/upgrade/copy_kick_system_images.yaml
+
+- include: targets/nxos_install_os/tasks/upgrade/install_with_kick.yaml
+  when: ki is defined and combined is undefined
+
+- include: targets/nxos_install_os/tasks/upgrade/install_system.yaml
+  when: combined is defined
+
+# Only needed when - meta: reset_connection does not work. Fixed in 2.6
+#- include: targets/nxos_install_os/tasks/upgrade/clear_persistent_sockets.yaml
+
+- meta: reset_connection
+
+- name: "Check installed OS for newly installed version {{ tv }}"
+  nxos_command:
+    commands: ['show version | json']
+  register: output
+
+- debug: msg="Version detected {{ output['stdout_lines'][0]['kickstart_ver_str'] }}"

test/integration/targets/nxos_install_os/tasks/upgrade/install_os_provider.yaml

@@ -0,0 +1,24 @@
+---
+- include: targets/nxos_install_os/tasks/upgrade/delete_files_provider.yaml
+  when: delete_image_list is defined
+
+- include: targets/nxos_install_os/tasks/upgrade/copy_kick_system_images.yaml
+
+- include: targets/nxos_install_os/tasks/upgrade/install_with_kick_provider.yaml
+  when: ki is defined and combined is undefined
+
+- include: targets/nxos_install_os/tasks/upgrade/install_system_provider.yaml
+  when: combined is defined
+
+# Only needed when - meta: reset_connection does not work. Fixed in 2.6
+#- include: targets/nxos_install_os/tasks/upgrade/clear_persistent_sockets.yaml
+
+- meta: reset_connection
+
+- name: "Check installed OS for newly installed version {{ tv }}"
+  nxos_command:
+    commands: ['show version | json']
+    provider: "{{ connection }}"
+  register: output
+
+- debug: msg="Version detected {{ output['stdout_lines'][0]['kickstart_ver_str'] }}"

test/integration/targets/nxos_install_os/tasks/upgrade/install_system.yaml

@@ -0,0 +1,24 @@
+---
+- name: "Install OS image {{ si }}"
+  check_mode: "{{ checkmode }}"
+  nxos_install_os:
+    system_image_file: "{{ si }}"
+    issu: "{{ issu }}"
+  register: result
+
+- debug: msg=" {{ result['install_state'] }}"
+
+- name: Wait for device to come back up with new image
+  wait_for:
+    port: 22
+    state: started
+    timeout: 500
+    delay: 60
+    host: "{{ inventory_hostname }}"
+  when: result.changed and not checkmode
+
+- debug: msg='Wait 5 mins to allow system to stabilize'
+  when: result.changed and not checkmode
+- pause:
+    seconds: 300
+  when: result.changed and not checkmode

test/integration/targets/nxos_install_os/tasks/upgrade/install_system_provider.yaml

@@ -0,0 +1,25 @@
+---
+- name: "Install OS image {{ si }} using provider"
+  check_mode: "{{ checkmode }}"
+  nxos_install_os:
+    system_image_file: "{{ si }}"
+    issu: "{{ issu }}"
+    provider: "{{ connection }}"
+  register: result
+
+- debug: msg=" {{ result['install_state'] }}"
+
+- name: Wait for device to come back up with new image
+  wait_for:
+    port: 22
+    state: started
+    timeout: 500
+    delay: 60
+    host: "{{ inventory_hostname }}"
+  when: result.changed and not checkmode
+
+- debug: msg='Wait 5 mins to allow system to stabilize'
+  when: result.changed and not checkmode
+- pause:
+    seconds: 300
+  when: result.changed and not checkmode

test/integration/targets/nxos_install_os/tasks/upgrade/install_with_kick.yaml

@@ -0,0 +1,25 @@
+---
+- name: "Install OS image {{ si }}"
+  check_mode: "{{ checkmode }}"
+  nxos_install_os:
+    system_image_file: "{{ si }}"
+    kickstart_image_file: "{{ ki }}"
+    issu: "{{ issu }}"
+  register: result
+
+- debug: msg=" {{ result['install_state'] }}"
+
+- name: Wait for device to come back up with new image
+  wait_for:
+    port: 22
+    state: started
+    timeout: 500
+    delay: 60
+    host: "{{ inventory_hostname }}"
+  when: result.changed and not checkmode
+
+- debug: msg='Wait 5 mins to allow system to stabilize'
+  when: result.changed and not checkmode
+- pause:
+    seconds: 300
+  when: result.changed and not checkmode

test/integration/targets/nxos_install_os/tasks/upgrade/install_with_kick_provider.yaml

@@ -0,0 +1,26 @@
+---
+- name: "Install OS image {{ si }} using provider"
+  check_mode: "{{ checkmode }}"
+  nxos_install_os:
+    system_image_file: "{{ si }}"
+    kickstart_image_file: "{{ ki }}"
+    issu: "{{ issu }}"
+    provider: "{{ connection }}"
+  register: result
+
+- debug: msg=" {{ result['install_state'] }}"
+
+- name: Wait for device to come back up with new image
+  wait_for:
+    port: 22
+    state: started
+    timeout: 500
+    delay: 60
+    host: "{{ inventory_hostname }}"
+  when: result.changed and not checkmode
+
+- debug: msg='Wait 5 mins to allow system to stabilize'
+  when: result.changed and not checkmode
+- pause:
+    seconds: 300
+  when: result.changed and not checkmode

test/integration/targets/nxos_install_os/tasks/upgrade/main_os_install.yaml

@@ -0,0 +1,10 @@
+---
+- debug: msg="***WARNING*** Remove meta end_play to verify this module ***WARNING***"
+
+- meta: end_play
+
+- include: targets/nxos_install_os/tasks/upgrade/install_os.yaml
+  when: connection is not defined
+
+- include: targets/nxos_install_os/tasks/upgrade/install_os_provider.yaml
+  when: connection is defined

test/integration/targets/nxos_install_os/tasks/upgrade/set_boot_pointer_and_reload.yaml

@@ -0,0 +1,22 @@
+---
+- name: "Reload to upgrade to OS image {{ si }}"
+  nxos_config:
+    lines:
+      - terminal dont-ask
+      - no boot nxos
+      - "boot nxos bootflash:{{ si }}"
+      - reload
+    match: none
+  ignore_errors: yes
+
+- name: Wait for device to come back up with new image
+  wait_for:
+    port: 22
+    state: started
+    timeout: 500
+    delay: 60
+    host: "{{ inventory_hostname }}"
+
+- debug: msg='Wait 5 mins to allow system to stabilize'
+- pause:
+    seconds: 300

test/integration/targets/nxos_install_os/tasks/upgrade/set_boot_pointer_and_reload_provider.yaml

@@ -0,0 +1,23 @@
+---
+- name: "Reload to upgrade to OS image {{ si }} using provider"
+  nxos_config:
+    lines:
+      - terminal dont-ask
+      - no boot nxos
+      - "boot nxos bootflash:{{ si }}"
+      - reload
+    match: none
+    provider: "{{ connection }}"
+  ignore_errors: yes
+
+- name: Wait for device to come back up with new image
+  wait_for:
+    port: 22
+    state: started
+    timeout: 500
+    delay: 60
+    host: "{{ inventory_hostname }}"
+
+- debug: msg='Wait 5 mins to allow system to stabilize'
+- pause:
+    seconds: 300

test/integration/targets/nxos_install_os/tests/common/upgrade.yaml

@@ -0,0 +1,56 @@
+---
+- debug: msg="START connection={{ ansible_connection }} nxos_os_install upgrade"
+- debug: msg="Using provider={{ connection.transport }}"
+  when: connection is defined
+
+- set_fact: checkmode='no'
+- set_fact: issu='desired'
+
+- set_fact: image_dir='/Users/mwiebe/Projects/nxos_ansible/images/'
+
+- set_fact:
+    delete_image_list:
+      - nxos.7.0.3.I7.2.bin
+      - nxos.7.0.3.I7.3.bin
+
+#---------------------------------------------------------#
+# Upgrade to 6.0(2)U6(1a)                                 #
+#---------------------------------------------------------#
+
+- set_fact: si='n3000-uk9.6.0.2.U6.1a.bin'
+- set_fact: ki='n3000-uk9-kickstart.6.0.2.U6.1a.bin'
+
+- name: Upgrade to U6.1a
+  include: targets/nxos_install_os/tasks/upgrade/main_os_install.yaml
+
+#---------------------------------------------------------#
+# Upgrade to 6.0(2)U6(2a)                                 #
+#---------------------------------------------------------#
+
+- set_fact: si='n3000-uk9.6.0.2.U6.2a.bin'
+- set_fact: ki='n3000-uk9-kickstart.6.0.2.U6.2a.bin'
+
+- name: Upgrade to U6.2a
+  include: targets/nxos_install_os/tasks/upgrade/main_os_install.yaml
+
+#---------------------------------------------------------#
+# Upgrade to 6.0(2)U6(3a)                                 #
+#---------------------------------------------------------#
+
+- set_fact: si='n3000-uk9.6.0.2.U6.3a.bin'
+- set_fact: ki='n3000-uk9-kickstart.6.0.2.U6.3a.bin'
+
+- name: Upgrade to U6.3a
+  include: targets/nxos_install_os/tasks/upgrade/main_os_install.yaml
+
+#---------------------------------------------------------#
+# Upgrade to 7.0(3)I7(3)                                  #
+#---------------------------------------------------------#
+
+- set_fact: si='nxos.7.0.3.I7.2.bin'
+- set_fact: combined='true'
+
+- name: Upgrade to 7.0.3.I7.2
+  include: targets/nxos_install_os/tasks/upgrade/main_os_install.yaml
+
+- debug: msg="END connection={{ ansible_connection }} nxos_os_install upgrade"