Update iosxr cliconf plugin (#43837)

* Update iosxr cliconf plugin

Fixes #39056

*  Update iosxr cliconf plugin
*  Modify iosxr module_utils code to support
   refactored cliconf plugin api's
*  Other minor changes

* Fix unit test failure

* Update ios, eos, nxos plugin for diff

* Fix review comment

lib/ansible/module_utils/network/ios/ios.py

@@ -132,8 +132,7 @@ def to_commands(module, commands):
 def run_commands(module, commands, check_rc=True):
     connection = get_connection(module)
     try:
-        out = connection.run_commands(commands=commands, check_rc=check_rc)
+        return connection.run_commands(commands=commands, check_rc=check_rc)
-        return out
     except ConnectionError as exc:
         module.fail_json(msg=to_text(exc))
 

lib/ansible/module_utils/network/iosxr/iosxr.py

@@ -199,8 +199,7 @@ def build_xml_subtree(container_ele, xmap, param=None, opcode=None):
 
 
 def build_xml(container, xmap=None, params=None, opcode=None):
-
+    """
-    '''
     Builds netconf xml rpc document from meta-data
 
     Args:
@@ -240,8 +239,7 @@ def build_xml(container, xmap=None, params=None, opcode=None):
               </banners>
             </config>
     :returns: xml rpc document as a string
-    '''
+    """
-
     if opcode == 'filter':
         root = etree.Element("filter", type="subtree")
     elif opcode in ('delete', 'merge'):
@@ -285,30 +283,17 @@ def etree_findall(root, node):
 
 def is_cliconf(module):
     capabilities = get_device_capabilities(module)
-    network_api = capabilities.get('network_api')
+    return True if capabilities.get('network_api') == 'cliconf' else False
-    if network_api not in ('cliconf', 'netconf'):
-        module.fail_json(msg=('unsupported network_api: {!s}'.format(network_api)))
-        return False
-
-    if network_api == 'cliconf':
-        return True
-
-    return False
 
 
 def is_netconf(module):
     capabilities = get_device_capabilities(module)
     network_api = capabilities.get('network_api')
-    if network_api not in ('cliconf', 'netconf'):
-        module.fail_json(msg=('unsupported network_api: {!s}'.format(network_api)))
-        return False
-
     if network_api == 'netconf':
         if not HAS_NCCLIENT:
-            module.fail_json(msg=('ncclient is not installed'))
+            module.fail_json(msg='ncclient is not installed')
         if not HAS_XML:
-            module.fail_json(msg=('lxml is not installed'))
+            module.fail_json(msg='lxml is not installed')
-
         return True
 
     return False
@@ -348,12 +333,15 @@ def commit_config(module, comment=None, confirmed=False, confirm_timeout=None,
     conn = get_connection(module)
     reply = None
     try:
+        if is_netconf(module):
             if check:
                 reply = conn.validate()
             else:
-            if is_netconf(module):
                 reply = conn.commit(confirmed=confirmed, timeout=confirm_timeout, persist=persist)
         elif is_cliconf(module):
+            if check:
+                module.fail_json(msg="Validate configuration is not supported with network_cli connection type")
+            else:
                 reply = conn.commit(comment=comment, label=label)
     except ConnectionError as exc:
         module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
@@ -380,10 +368,10 @@ def get_config(module, config_filter=None, source='running'):
 
     # Note: Does not cache config in favour of latest config on every get operation.
     try:
-        out = conn.get_config(source=source, filter=config_filter)
         if is_netconf(module):
             out = to_xml(conn.get_config(source=source, filter=config_filter))
-
+        elif is_cliconf(module):
+            out = conn.get_config(source=source, flags=config_filter)
         cfg = out.strip()
     except ConnectionError as exc:
         module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
@@ -429,10 +417,6 @@ def load_config(module, command_filter, commit=False, replace=False,
             pass
 
     elif is_cliconf(module):
-        # to keep the pre-cliconf behaviour, make a copy, avoid adding commands to input list
-        cmd_filter = deepcopy(command_filter)
-        # If label is present check if label already exist before entering
-        # config mode
         try:
             if label:
                 old_label = check_existing_commit_labels(conn, label)
@@ -442,68 +426,23 @@ def load_config(module, command_filter, commit=False, replace=False,
                         ' an earlier commit, please choose a different label'
                         ' and rerun task' % label
                     )
-            cmd_filter.insert(0, 'configure terminal')
-            if admin:
-                cmd_filter.insert(0, 'admin')
 
-            conn.edit_config(cmd_filter)
+            response = conn.edit_config(candidate=command_filter, commit=commit, admin=admin, replace=replace, comment=comment, label=label)
             if module._diff:
-                diff = get_config_diff(module)
+                diff = response.get('diff')
-
-            if replace:
-                cmd = list()
-                cmd.append({'command': 'commit replace',
-                            'prompt': 'This commit will replace or remove the entire running configuration',
-                            'answer': 'yes'})
-                cmd.append('end')
-                conn.edit_config(cmd)
-            elif commit:
-                commit_config(module, comment=comment, label=label)
-                conn.edit_config('end')
-                if admin:
-                    conn.edit_config('exit')
-            else:
-                conn.discard_changes()
         except ConnectionError as exc:
             module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
 
     return diff
 
 
-def run_command(module, commands):
+def run_commands(module, commands, check_rc=True):
-    conn = get_connection(module)
+    connection = get_connection(module)
-    responses = list()
-    for cmd in to_list(commands):
-
     try:
-            if isinstance(cmd, str):
+        return connection.run_commands(commands=commands, check_rc=check_rc)
-                cmd = json.loads(cmd)
-            command = cmd.get('command', None)
-            prompt = cmd.get('prompt', None)
-            answer = cmd.get('answer', None)
-            sendonly = cmd.get('sendonly', False)
-            newline = cmd.get('newline', True)
-        except:
-            command = cmd
-            prompt = None
-            answer = None
-            sendonly = False
-            newline = True
-
-        try:
-            out = conn.get(command=command, prompt=prompt, answer=answer, sendonly=sendonly, newline=newline)
     except ConnectionError as exc:
         module.fail_json(msg=to_text(exc))
 
-        try:
-            out = to_text(out, errors='surrogate_or_strict')
-        except UnicodeError:
-            module.fail_json(msg=u'Failed to decode output from {0}: {1}'.format(cmd, to_text(out)))
-
-        responses.append(out)
-
-    return responses
-
 
 def copy_file(module, src, dst, proto='scp'):
     conn = get_connection(module)

lib/ansible/modules/network/iosxr/iosxr_command.py

@@ -122,7 +122,7 @@ failed_conditions:
 import time
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.iosxr.iosxr import run_command, iosxr_argument_spec
+from ansible.module_utils.network.iosxr.iosxr import run_commands, iosxr_argument_spec
 from ansible.module_utils.network.iosxr.iosxr import command_spec
 from ansible.module_utils.network.common.parsing import Conditional
 from ansible.module_utils.six import string_types
@@ -188,7 +188,7 @@ def main():
     match = module.params['match']
 
     while retries > 0:
-        responses = run_command(module, commands)
+        responses = run_commands(module, commands)
 
         for item in list(conditionals):
             if item(responses):

lib/ansible/modules/network/iosxr/iosxr_config.py

@@ -366,6 +366,8 @@ def run(module, result):
     running_config = get_running_config(module)
 
     commands = None
+    replace_file_path = None
+
     if match != 'none' and replace != 'config':
         commands = candidate_config.difference(running_config, path=path, match=match, replace=replace)
     elif replace_config:
@@ -380,6 +382,7 @@ def run(module, result):
                 module.fail_json(msg='Copy of config file to the node failed')
 
             commands = ['load harddisk:/ansible_config.txt']
+            replace_file_path = 'harddisk:/ansible_config.txt'
     else:
         commands = candidate_config.items
 
@@ -399,7 +402,7 @@ def run(module, result):
         commit = not check_mode
         diff = load_config(
             module, commands, commit=commit,
-            replace=replace_config, comment=comment, admin=admin,
+            replace=replace_file_path, comment=comment, admin=admin,
             label=label
         )
         if diff:

lib/ansible/modules/network/iosxr/iosxr_facts.py

@@ -118,7 +118,7 @@ ansible_net_neighbors:
 import re
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.iosxr.iosxr import iosxr_argument_spec, run_command
+from ansible.module_utils.network.iosxr.iosxr import iosxr_argument_spec, run_commands
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.six.moves import zip
 
@@ -407,7 +407,7 @@ def main():
     try:
         for inst in instances:
             commands = inst.commands()
-            responses = run_command(module, commands)
+            responses = run_commands(module, commands)
             results = dict(zip(commands, responses))
             inst.populate(results)
             facts.update(inst.facts)

lib/ansible/modules/network/iosxr/iosxr_interface.py

@@ -195,7 +195,7 @@ import collections
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.iosxr.iosxr import get_config, load_config, build_xml
-from ansible.module_utils.network.iosxr.iosxr import run_command, iosxr_argument_spec, get_oper
+from ansible.module_utils.network.iosxr.iosxr import run_commands, iosxr_argument_spec, get_oper
 from ansible.module_utils.network.iosxr.iosxr import is_netconf, is_cliconf, etree_findall, etree_find
 from ansible.module_utils.network.common.utils import conditional, remove_default_spec
 
@@ -382,7 +382,7 @@ class CliConfiguration(ConfigBase):
                 sleep(want_item['delay'])
 
             command = 'show interfaces {!s}'.format(want_item['name'])
-            out = run_command(self._module, command)[0]
+            out = run_commands(self._module, command)[0]
 
             if want_state in ('up', 'down'):
                 match = re.search(r'%s (\w+)' % 'line protocol is', out, re.M)

lib/ansible/plugins/cliconf/__init__.py

@@ -175,7 +175,7 @@ class CliconfBase(AnsiblePlugin):
         :param source: The configuration source to return from the device.
             This argument accepts either `running` or `startup` as valid values.
 
-        :param flag: For devices that support configuration filtering, this
+        :param flags: For devices that support configuration filtering, this
             keyword argument is used to filter the returned configuration.
             The use of this keyword argument is device dependent adn will be
             silently ignored on devices that do not support it.
@@ -212,7 +212,8 @@ class CliconfBase(AnsiblePlugin):
                  response on executing configuration commands and platform relevant data.
                {
                    "diff": "",
-                   "response": []
+                   "response": [],
+                   "request": []
                }
 
         """
@@ -267,6 +268,8 @@ class CliconfBase(AnsiblePlugin):
                     'supports_diff_match: <bool>,          # identify if match is supported
                     'supports_diff_ignore_lines: <bool>,   # identify if ignore line in diff is supported
                     'supports_config_replace': <bool>,     # identify if running config replace with candidate config is supported
+                    'supports_admin': <bool>,              # identify if admin configure mode is supported or not
+                    'supports_commit_label': <bool>,       # identify if commit label is supported or not
                 }
                 'format': [list of supported configuration format],
                 'diff_match': [list of supported match values],

lib/ansible/plugins/cliconf/eos.py

@@ -214,7 +214,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'replace' value %s in invalid, valid values are %s" % (diff_replace, ', '.join(option_values['diff_replace'])))
 
         # prepare candidate configuration
-        candidate_obj = NetworkConfig(indent=3)
+        candidate_obj = NetworkConfig(indent=3, ignore_lines=diff_ignore_lines)
         candidate_obj.load(candidate)
 
         if running and diff_match != 'none' and diff_replace != 'config':

lib/ansible/plugins/cliconf/ios.py

@@ -105,7 +105,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'replace' value %s in invalid, valid values are %s" % (diff_replace, ', '.join(option_values['diff_replace'])))
 
         # prepare candidate configuration
-        candidate_obj = NetworkConfig(indent=1)
+        candidate_obj = NetworkConfig(indent=1, ignore_lines=diff_ignore_lines)
         want_src, want_banners = self._extract_banners(candidate)
         candidate_obj.load(want_src)
 

lib/ansible/plugins/cliconf/iosxr.py

@@ -19,12 +19,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import collections
 import re
 import json
 
-from itertools import chain
+from ansible.errors import AnsibleConnectionFailure
-
 from ansible.module_utils._text import to_text
+from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase
 
@@ -56,56 +57,167 @@ class Cliconf(CliconfBase):
 
         return device_info
 
-    def get_config(self, source='running', format='text', filter=None):
+    def configure(self, admin=False):
+        prompt = to_text(self._connection.get_prompt(), errors='surrogate_or_strict').strip()
+        if not prompt.endswith(')#'):
+            if admin and 'admin-' not in prompt:
+                self.send_command('admin')
+            self.send_command('configure terminal')
+
+    def abort(self, admin=False):
+        prompt = to_text(self._connection.get_prompt(), errors='surrogate_or_strict').strip()
+        if prompt.endswith(')#'):
+            self.send_command('abort')
+            if admin and 'admin-' in prompt:
+                self.send_command('exit')
+
+    def get_config(self, source='running', format='text', flags=None):
+        if source not in ['running']:
+            raise ValueError("fetching configuration from %s is not supported" % source)
+
         lookup = {'running': 'running-config'}
-        if source not in lookup:
+
-            return self.invalid_params("fetching configuration from %s is not supported" % source)
-        if filter:
-            cmd = 'show {0} {1}'.format(lookup[source], filter)
-        else:
         cmd = 'show {0} '.format(lookup[source])
+        cmd += ' '.join(to_list(flags))
+        cmd = cmd.strip()
 
         return self.send_command(cmd)
 
-    def edit_config(self, commands=None):
+    def edit_config(self, candidate=None, commit=True, admin=False, replace=None, comment=None, label=None):
-        for cmd in chain(to_list(commands)):
+        operations = self.get_device_operations()
-            try:
+        self.check_edit_config_capabiltiy(operations, candidate, commit, replace, comment)
-                if isinstance(cmd, str):
-                    cmd = json.loads(cmd)
-                command = cmd.get('command', None)
-                prompt = cmd.get('prompt', None)
-                answer = cmd.get('answer', None)
-                sendonly = cmd.get('sendonly', False)
-                newline = cmd.get('newline', True)
-            except:
-                command = cmd
-                prompt = None
-                answer = None
-                sendonly = None
-                newline = None
 
-            self.send_command(command=command, prompt=prompt, answer=answer, sendonly=sendonly, newline=newline)
+        resp = {}
+        results = []
+        requests = []
+
+        self.configure(admin=admin)
+
+        if replace:
+            candidate = 'load {0}'.format(replace)
+
+        for line in to_list(candidate):
+            if not isinstance(line, collections.Mapping):
+                line = {'command': line}
+            cmd = line['command']
+            results.append(self.send_command(**line))
+            requests.append(cmd)
+
+        diff = self.get_diff(admin=admin)
+        config_diff = diff.get('config_diff')
+        if config_diff or replace:
+            resp['diff'] = config_diff
+            if commit:
+                self.commit(comment=comment, label=label, replace=replace)
+            else:
+                self.discard_changes()
+
+        self.abort(admin=admin)
+
+        resp['request'] = requests
+        resp['response'] = results
+        return resp
+
+    def get_diff(self, admin=False):
+        self.configure(admin=admin)
+
+        diff = {'config_diff': None}
+        response = self.send_command('show commit changes diff')
+        for item in response.splitlines():
+            if item and item[0] in ['<', '+', '-']:
+                diff['config_diff'] = response
+                break
+        return diff
 
     def get(self, command=None, prompt=None, answer=None, sendonly=False, newline=True, output=None):
+        if output:
+            raise ValueError("'output' value %s is not supported for get" % output)
         return self.send_command(command=command, prompt=prompt, answer=answer, sendonly=sendonly, newline=newline)
 
-    def commit(self, comment=None, label=None):
+    def commit(self, comment=None, label=None, replace=None):
-        if comment and label:
+        cmd_obj = {}
-            command = 'commit label {0} comment {1}'.format(label, comment)
+        if replace:
-        elif comment:
+            cmd_obj['command'] = 'commit replace'
-            command = 'commit comment {0}'.format(comment)
+            cmd_obj['prompt'] = 'This commit will replace or remove the entire running configuration'
-        elif label:
+            cmd_obj['answer'] = 'yes'
-            command = 'commit label {0}'.format(label)
         else:
-            command = 'commit'
+            if comment and label:
-        self.send_command(command)
+                cmd_obj['command'] = 'commit label {0} comment {1}'.format(label, comment)
+            elif comment:
+                cmd_obj['command'] = 'commit comment {0}'.format(comment)
+            elif label:
+                cmd_obj['command'] = 'commit label {0}'.format(label)
+            else:
+                cmd_obj['command'] = 'commit'
+
+        self.send_command(**cmd_obj)
+
+    def run_commands(self, commands=None, check_rc=True):
+        if commands is None:
+            raise ValueError("'commands' value is required")
+        responses = list()
+        for cmd in to_list(commands):
+            if not isinstance(cmd, collections.Mapping):
+                cmd = {'command': cmd}
+
+            output = cmd.pop('output', None)
+            if output:
+                raise ValueError("'output' value %s is not supported for run_commands" % output)
+
+            try:
+                out = self.send_command(**cmd)
+            except AnsibleConnectionFailure as e:
+                if check_rc:
+                    raise
+                out = getattr(e, 'err', e)
+
+            if out is not None:
+                try:
+                    out = to_text(out, errors='surrogate_or_strict').strip()
+                except UnicodeError:
+                    raise ConnectionError(msg=u'Failed to decode output from %s: %s' % (cmd, to_text(out)))
+
+                try:
+                    out = json.loads(out)
+                except ValueError:
+                    pass
+
+                responses.append(out)
+        return responses
 
     def discard_changes(self):
         self.send_command('abort')
 
+    def get_device_operations(self):
+        return {
+            'supports_diff_replace': False,
+            'supports_commit': True,
+            'supports_rollback': True,
+            'supports_defaults': False,
+            'supports_onbox_diff': True,
+            'supports_commit_comment': True,
+            'supports_multiline_delimiter': False,
+            'supports_diff_match': False,
+            'supports_diff_ignore_lines': False,
+            'supports_generate_diff': False,
+            'supports_replace': True,
+            'supports_admin': True,
+            'supports_commit_label': True
+        }
+
+    def get_option_values(self):
+        return {
+            'format': ['text'],
+            'diff_match': [],
+            'diff_replace': [],
+            'output': []
+        }
+
     def get_capabilities(self):
         result = {}
-        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes']
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'get_diff', 'configure', 'exit']
         result['network_api'] = 'cliconf'
         result['device_info'] = self.get_device_info()
+        result['device_operations'] = self.get_device_operations()
+        result.update(self.get_option_values())
         return json.dumps(result)

lib/ansible/plugins/cliconf/junos.py

@@ -115,6 +115,10 @@ class Cliconf(CliconfBase):
             else:
                 self.discard_changes()
 
+        else:
+            for cmd in ['top', 'exit']:
+                self.send_command(cmd)
+
         resp['request'] = requests
         resp['response'] = results
         return resp
@@ -166,7 +170,11 @@ class Cliconf(CliconfBase):
         return resp
 
     def get_diff(self, rollback_id=None):
-        return self.compare_configuration(rollback_id=rollback_id)
+        diff = {'config_diff': None}
+        response = self.compare_configuration(rollback_id=rollback_id)
+        if response:
+            diff['config_diff'] = response
+        return diff
 
     def get_device_operations(self):
         return {

lib/ansible/plugins/cliconf/nxos.py

@@ -115,7 +115,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'replace' value %s in invalid, valid values are %s" % (diff_replace, ', '.join(option_values['diff_replace'])))
 
         # prepare candidate configuration
-        candidate_obj = NetworkConfig(indent=2)
+        candidate_obj = NetworkConfig(indent=2, ignore_lines=diff_ignore_lines)
         candidate_obj.load(candidate)
 
         if running and diff_match != 'none' and diff_replace != 'config':
@@ -215,7 +215,7 @@ class Cliconf(CliconfBase):
                 try:
                     out = json.loads(out)
                 except ValueError:
-                    out = to_text(out, errors='surrogate_or_strict').strip()
+                    pass
 
                 responses.append(out)
         return responses

test/units/modules/network/iosxr/test_iosxr_command.py

@@ -32,13 +32,13 @@ class TestIosxrCommandModule(TestIosxrModule):
     def setUp(self):
         super(TestIosxrCommandModule, self).setUp()
 
-        self.mock_run_command = patch('ansible.modules.network.iosxr.iosxr_command.run_command')
+        self.mock_run_commands = patch('ansible.modules.network.iosxr.iosxr_command.run_commands')
-        self.run_command = self.mock_run_command.start()
+        self.run_commands = self.mock_run_commands.start()
 
     def tearDown(self):
         super(TestIosxrCommandModule, self).tearDown()
 
-        self.mock_run_command.stop()
+        self.mock_run_commands.stop()
 
     def load_fixtures(self, commands=None):
 
@@ -55,7 +55,7 @@ class TestIosxrCommandModule(TestIosxrModule):
                 output.append(load_fixture(filename))
             return output
 
-        self.run_command.side_effect = load_from_file
+        self.run_commands.side_effect = load_from_file
 
     def test_iosxr_command_simple(self):
         set_module_args(dict(commands=['show version']))
@@ -78,13 +78,13 @@ class TestIosxrCommandModule(TestIosxrModule):
         wait_for = 'result[0] contains "test string"'
         set_module_args(dict(commands=['show version'], wait_for=wait_for))
         self.execute_module(failed=True)
-        self.assertEqual(self.run_command.call_count, 10)
+        self.assertEqual(self.run_commands.call_count, 10)
 
     def test_iosxr_command_retries(self):
         wait_for = 'result[0] contains "test string"'
         set_module_args(dict(commands=['show version'], wait_for=wait_for, retries=2))
         self.execute_module(failed=True)
-        self.assertEqual(self.run_command.call_count, 2)
+        self.assertEqual(self.run_commands.call_count, 2)
 
     def test_iosxr_command_match_any(self):
         wait_for = ['result[0] contains "Cisco IOS"',

test/units/modules/network/iosxr/test_iosxr_facts.py

@@ -34,14 +34,14 @@ class TestIosxrFacts(TestIosxrModule):
     def setUp(self):
         super(TestIosxrFacts, self).setUp()
 
-        self.mock_run_command = patch(
+        self.mock_run_commands = patch(
-            'ansible.modules.network.iosxr.iosxr_facts.run_command')
+            'ansible.modules.network.iosxr.iosxr_facts.run_commands')
-        self.run_command = self.mock_run_command.start()
+        self.run_commands = self.mock_run_commands.start()
 
     def tearDown(self):
         super(TestIosxrFacts, self).tearDown()
 
-        self.mock_run_command.stop()
+        self.mock_run_commands.stop()
 
     def load_fixtures(self, commands=None):
 
@@ -61,7 +61,7 @@ class TestIosxrFacts(TestIosxrModule):
                 output.append(load_fixture(filename))
             return output
 
-        self.run_command.side_effect = load_from_file
+        self.run_commands.side_effect = load_from_file
 
     def test_iosxr_facts_gather_subset_default(self):
         set_module_args(dict())