network_put and network_get modules (#39592)

* Initial commit

* Socket Timeout and dest file handler

* sftp handling

* module name change as per review

* multiple thread tmp file overwite problem

* Integration test suite for network_put

* add additional testcase for dest argument

* fix pylint/pep8/modules warnings

* add socket timeout for get_file

* network_get module

* pep8 issue on network_get

* Review comments

lib/ansible/modules/network/files/network_get.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2018, Ansible by Red Hat, inc
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+
+DOCUMENTATION = """
+---
+module: network_get
+version_added: "2.6"
+author: "Deepak Agrawal (@dagrawal)"
+short_description: Copy files from a network device to Ansible Controller
+description:
+  - This module provides functionlity to copy file from network device to
+    ansible controller.
+options:
+  src:
+    description:
+      - Specifies the source file. The path to the source file can either be
+        the full path on the network device or a relative path as per path
+        supported by destination network device.
+    required: true
+  protocol:
+    description:
+      - Protocol used to transfer file.
+    default: scp
+    choices: ['scp', 'sftp']
+  dest:
+    description:
+      - Specifies the destination file. The path to the destination file can
+        either be the full path on the Ansible control host or a relative
+        path from the playbook or role root directory.
+    default:
+      - Same filename as specified in src. The path will be playbook root
+        or role root directory if playbook is part of a role.
+
+requirements:
+    - "scp"
+
+notes:
+   - Some devices need specific configurations to be enabled before scp can work
+     These configuration should be pre-configued before using this module
+     e.g ios - C(ip scp server enable)
+   - User privileage to do scp on network device should be pre-configured
+     e.g. ios - need user privileage 15 by default for allowing scp
+   - Default destination of source file
+"""
+
+EXAMPLES = """
+- name: copy file from the network device to ansible controller
+  network_get:
+    src: running_cfg_ios1.txt
+
+- name: copy file from ios to common location at /tmp
+  network_put:
+    src: running_cfg_sw1.txt
+    dest : /tmp/ios1.txt
+"""
+
+RETURN = """
+"""

lib/ansible/modules/network/files/network_put.py

@@ -0,0 +1,71 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2018, Ansible by Red Hat, inc
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+
+DOCUMENTATION = """
+---
+module: network_put
+version_added: "2.6"
+author: "Deepak Agrawal (@dagrawal)"
+short_description: Copy files from Ansibe controller to a network device
+description:
+  - This module provides functionlity to copy file from Ansible controller to
+    network devices.
+options:
+  src:
+    description:
+      - Specifies the source file. The path to the source file can either be
+        the full path on the Ansible control host or a relative path from the
+        playbook or role root directory.
+    required: true
+  protocol:
+    description:
+      - Protocol used to transfer file.
+    default: scp
+    choices: ['scp', 'sftp']
+  dest:
+    description:
+      - Specifies the destination file. The path to destination file can
+        either be the full path or relative path as supported by network_os.
+    default:
+      - Filename from src and at default directory of user shell on
+        network_os.
+    required: no
+
+requirements:
+    - "scp"
+
+notes:
+   - Some devices need specific configurations to be enabled before scp can work
+     These configuration should be pre-configued before using this module
+     e.g ios - C(ip scp server enable).
+   - User privileage to do scp on network device should be pre-configured
+     e.g. ios - need user privileage 15 by default for allowing scp.
+   - Default destination of source file.
+"""
+
+EXAMPLES = """
+- name: copy file from ansible controller to a network device
+  network_put:
+    src: running_cfg_ios1.txt
+
+- name: copy file at root dir of flash in slot 3 of sw1(ios)
+  network_put:
+    src: running_cfg_sw1.txt
+    protocol: sftp
+    dest : flash3:/running_cfg_sw1.txt
+"""
+
+RETURN = """
+"""

lib/ansible/plugins/action/network_get.py

@@ -0,0 +1,130 @@
+# (c) 2018, Ansible Inc,
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import copy
+import os
+import time
+import re
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.connection import Connection
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
+from ansible.module_utils.six.moves.urllib.parse import urlsplit
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=None):
+        socket_path = None
+        play_context = copy.deepcopy(self._play_context)
+        play_context.network_os = self._get_network_os(task_vars)
+
+        result = super(ActionModule, self).run(task_vars=task_vars)
+
+        if play_context.connection != 'network_cli':
+            # It is supported only with network_cli
+            result['failed'] = True
+            result['msg'] = ('please use network_cli connection type for network_get module')
+            return result
+
+        try:
+            src = self._task.args.get('src')
+        except KeyError as exc:
+            return {'failed': True, 'msg': 'missing required argument: %s' % exc}
+
+        # Get destination file if specified
+        dest = self._task.args.get('dest')
+
+        if dest is None:
+            dest = self._get_default_dest(src)
+        else:
+            dest = self._handle_dest_path(dest)
+
+        # Get proto
+        proto = self._task.args.get('protocol')
+        if proto is None:
+            proto = 'scp'
+
+        sock_timeout = play_context.timeout
+
+        if socket_path is None:
+            socket_path = self._connection.socket_path
+
+        conn = Connection(socket_path)
+
+        try:
+            out = conn.get_file(
+                source=src, destination=dest,
+                proto=proto, timeout=sock_timeout
+            )
+        except Exception as exc:
+            result['failed'] = True
+            result['msg'] = ('Exception received : %s' % exc)
+
+        result['changed'] = True
+        result['destination'] = dest
+        return result
+
+    def _handle_dest_path(self, dest):
+        working_path = self._get_working_path()
+
+        if os.path.isabs(dest) or urlsplit('dest').scheme:
+            dst = dest
+        else:
+            dst = self._loader.path_dwim_relative(working_path, '', dest)
+
+        return dst
+
+    def _get_src_filename_from_path(self, src_path):
+        filename_list = re.split('/|:', src_path)
+        return filename_list[-1]
+
+    def _get_working_path(self):
+        cwd = self._loader.get_basedir()
+        if self._task._role is not None:
+            cwd = self._task._role._role_path
+        return cwd
+
+    def _get_default_dest(self, src_path):
+        dest_path = self._get_working_path()
+        src_fname = self._get_src_filename_from_path(src_path)
+        filename = '%s/%s' % (dest_path, src_fname)
+        return filename
+
+    def _get_network_os(self, task_vars):
+        if 'network_os' in self._task.args and self._task.args['network_os']:
+            display.vvvv('Getting network OS from task argument')
+            network_os = self._task.args['network_os']
+        elif self._play_context.network_os:
+            display.vvvv('Getting network OS from inventory')
+            network_os = self._play_context.network_os
+        elif 'network_os' in task_vars.get('ansible_facts', {}) and task_vars['ansible_facts']['network_os']:
+            display.vvvv('Getting network OS from fact')
+            network_os = task_vars['ansible_facts']['network_os']
+        else:
+            raise AnsibleError('ansible_network_os must be specified on this host to use platform agnostic modules')
+
+        return network_os

lib/ansible/plugins/action/network_put.py

@@ -0,0 +1,167 @@
+# (c) 2018, Ansible Inc,
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import copy
+import os
+import time
+import uuid
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.connection import Connection
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
+from ansible.module_utils.six.moves.urllib.parse import urlsplit
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=None):
+        socket_path = None
+        play_context = copy.deepcopy(self._play_context)
+        play_context.network_os = self._get_network_os(task_vars)
+
+        result = super(ActionModule, self).run(task_vars=task_vars)
+
+        if play_context.connection != 'network_cli':
+            # It is supported only with network_cli
+            result['failed'] = True
+            result['msg'] = ('please use network_cli connection type for network_put module')
+            return result
+
+        src_file_path_name = self._task.args.get('src')
+
+        try:
+            self._handle_template()
+        except ValueError as exc:
+            return dict(failed=True, msg=to_text(exc))
+
+        try:
+            src = self._task.args.get('src')
+        except KeyError as exc:
+            return {'failed': True, 'msg': 'missing required argument: %s' % exc}
+
+        # Get destination file if specified
+        dest = self._task.args.get('dest')
+
+        # Get proto
+        proto = self._task.args.get('protocol')
+        if proto is None:
+            proto = 'scp'
+
+        sock_timeout = play_context.timeout
+
+        # Now src has resolved file write to disk in current diectory for scp
+        filename = str(uuid.uuid4())
+        cwd = self._loader.get_basedir()
+        output_file = cwd + '/' + filename
+        with open(output_file, 'w') as f:
+            f.write(src)
+
+        if socket_path is None:
+            socket_path = self._connection.socket_path
+
+        conn = Connection(socket_path)
+        if dest is None:
+            dest = src_file_path_name
+
+        try:
+            out = conn.copy_file(
+                source=output_file, destination=dest,
+                proto=proto, timeout=sock_timeout
+            )
+        except Exception as exc:
+            if to_text(exc) == "No response from server":
+                if play_context.network_os == 'iosxr':
+                    # IOSXR sometimes closes socket prematurely after completion
+                    # of file transfer
+                    result['msg'] = 'Warning: iosxr scp server pre close issue. Please check dest'
+            else:
+                result['failed'] = True
+                result['msg'] = ('Exception received : %s' % exc)
+
+        # Cleanup tmp file expanded wih ansible vars
+        os.remove(output_file)
+        result['changed'] = True
+        return result
+
+    def _get_working_path(self):
+        cwd = self._loader.get_basedir()
+        if self._task._role is not None:
+            cwd = self._task._role._role_path
+        return cwd
+
+    def _handle_template(self):
+        src = self._task.args.get('src')
+        working_path = self._get_working_path()
+
+        if os.path.isabs(src) or urlsplit('src').scheme:
+            source = src
+        else:
+            source = self._loader.path_dwim_relative(working_path, 'templates', src)
+            if not source:
+                source = self._loader.path_dwim_relative(working_path, src)
+
+        if not os.path.exists(source):
+            raise ValueError('path specified in src not found')
+
+        try:
+            with open(source, 'r') as f:
+                template_data = to_text(f.read())
+        except IOError:
+            return dict(failed=True, msg='unable to load src file')
+
+        # Create a template search path in the following order:
+        # [working_path, self_role_path, dependent_role_paths, dirname(source)]
+        searchpath = [working_path]
+        if self._task._role is not None:
+            searchpath.append(self._task._role._role_path)
+            if hasattr(self._task, "_block:"):
+                dep_chain = self._task._block.get_dep_chain()
+                if dep_chain is not None:
+                    for role in dep_chain:
+                        searchpath.append(role._role_path)
+        searchpath.append(os.path.dirname(source))
+        self._templar.environment.loader.searchpath = searchpath
+        self._task.args['src'] = self._templar.template(
+            template_data,
+            convert_data=False
+        )
+
+        return dict(failed=False, msg='successfully loaded file')
+
+    def _get_network_os(self, task_vars):
+        if 'network_os' in self._task.args and self._task.args['network_os']:
+            display.vvvv('Getting network OS from task argument')
+            network_os = self._task.args['network_os']
+        elif self._play_context.network_os:
+            display.vvvv('Getting network OS from inventory')
+            network_os = self._play_context.network_os
+        elif 'network_os' in task_vars.get('ansible_facts', {}) and task_vars['ansible_facts']['network_os']:
+            display.vvvv('Getting network OS from fact')
+            network_os = task_vars['ansible_facts']['network_os']
+        else:
+            raise AnsibleError('ansible_network_os must be specified on this host to use platform agnostic modules')
+
+        return network_os

lib/ansible/plugins/cliconf/__init__.py

@@ -178,25 +178,25 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         "Discard changes in candidate datastore"
         return self._connection.method_not_found("discard_changes is not supported by network_os %s" % self._play_context.network_os)
 
-    def copy_file(self, source=None, destination=None, proto='scp'):
+    def copy_file(self, source=None, destination=None, proto='scp', timeout=30):
         """Copies file over scp/sftp to remote device"""
         ssh = self._connection.paramiko_conn._connect_uncached()
         if proto == 'scp':
             if not HAS_SCP:
                 self._connection.internal_error("Required library scp is not installed.  Please install it using `pip install scp`")
-            with SCPClient(ssh.get_transport()) as scp:
+            with SCPClient(ssh.get_transport(), socket_timeout=timeout) as scp:
-                scp.put(source, destination)
+                out = scp.put(source, destination)
         elif proto == 'sftp':
             with ssh.open_sftp() as sftp:
                 sftp.put(source, destination)
 
-    def get_file(self, source=None, destination=None, proto='scp'):
+    def get_file(self, source=None, destination=None, proto='scp', timeout=30):
         """Fetch file over scp/sftp from remote device"""
         ssh = self._connection.paramiko_conn._connect_uncached()
         if proto == 'scp':
             if not HAS_SCP:
                 self._connection.internal_error("Required library scp is not installed.  Please install it using `pip install scp`")
-            with SCPClient(ssh.get_transport()) as scp:
+            with SCPClient(ssh.get_transport(), socket_timeout=timeout) as scp:
                 scp.get(source, destination)
         elif proto == 'sftp':
             with ssh.open_sftp() as sftp:

test/integration/targets/ios_file/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+testcase: "*"

test/integration/targets/ios_file/ios1.cfg

@@ -0,0 +1,3 @@
+vlan 3
+ name ank_vlan3
+!

test/integration/targets/ios_file/tasks/cli.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all cli test cases
+  find:
+    paths: "{{ role_path }}/tests/cli"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  delegate_to: localhost
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test cases (connection=network_cli)
+  include: "{{ test_case_to_run }}"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/ios_file/tasks/main.yaml

@@ -0,0 +1,2 @@
+---
+- { include: cli.yaml, tags: ['cli'] }

test/integration/targets/ios_file/tests/cli/network_get.yaml

@@ -0,0 +1,43 @@
+---
+- debug: msg="START ios cli/network_get.yaml on connection={{ ansible_connection }}"
+
+# Add minimal testcase to check args are passed correctly to
+# implementation module and module run is successful.
+
+- name: setup
+  ios_config:
+    lines:
+      - ip ssh version 2
+      - ip scp server enable
+      - username {{ ansible_ssh_user }} privilege 15
+    match: none
+
+- name: setup (copy file to be fetched from device)
+  network_put:
+    src: ios1.cfg
+  register: result
+
+- assert:
+    that:
+      - result.changed == true
+
+- name: get the file from device with dest unspecified
+  network_get:
+    src: ios1.cfg
+  register: result
+
+- assert:
+    that:
+      - result.changed == true
+
+- name: get the file from device with relative destination
+  network_get:
+    src: ios1.cfg
+    dest: 'ios_{{ ansible_host }}.cfg'
+  register: result
+
+- assert:
+    that:
+      - result.changed == true
+
+- debug: msg="END ios cli/network_get.yaml on connection={{ ansible_connection }}"

test/integration/targets/ios_file/tests/cli/network_put.yaml

@@ -0,0 +1,34 @@
+---
+- debug: msg="START ios cli/network_put.yaml on connection={{ ansible_connection }}"
+
+# Add minimal testcase to check args are passed correctly to
+# implementation module and module run is successful.
+
+- name: setup
+  ios_config:
+    lines:
+      - ip ssh version 2
+      - ip scp server enable
+      - username {{ ansible_ssh_user }} privilege 15
+    match: none
+
+- name: copy file from controller to ios + scp (Default)
+  network_put:
+    src: ios1.cfg
+  register: result
+
+- assert:
+    that:
+      - result.changed == true
+
+- name: copy file from controller to ios + dest specified
+  network_put:
+    src: ios1.cfg
+    dest: ios.cfg
+  register: result
+
+- assert:
+    that:
+      - result.changed == true
+
+- debug: msg="END ios cli/network_put.yaml on connection={{ ansible_connection }}"