Add run_commands api for ios and vyos cliconf plugin (#42093)

* Add run_commands api for ios and vyos cliconf plugin

*  Add run_commands api to ios and vyos cliconf plugin
*  Refactor ios and vyos module_utils to check return code
   in run_commands

* Fix Ci failures

lib/ansible/module_utils/network/ios/ios.py

@@ -134,31 +134,21 @@ def run_commands(module, commands, check_rc=True):
     responses = list()
     connection = get_connection(module)
 
-    for cmd in to_list(commands):
+    try:
-        if isinstance(cmd, dict):
+        outputs = connection.run_commands(commands)
-            command = cmd['command']
+    except ConnectionError as exc:
-            prompt = cmd['prompt']
+        if check_rc:
-            answer = cmd['answer']
+            module.fail_json(msg=to_text(exc))
         else:
-            command = cmd
+            outputs = exc
-            prompt = None
-            answer = None
 
+    for item in to_list(outputs):
         try:
-            out = connection.get(command, prompt, answer)
+            item = to_text(item, errors='surrogate_or_strict')
-        except ConnectionError as exc:
-            if check_rc:
-                module.fail_json(msg=to_text(exc))
-            else:
-                out = exc
-
-        try:
-            out = to_text(out, errors='surrogate_or_strict')
         except UnicodeError:
-            module.fail_json(msg=u'Failed to decode output from %s: %s' % (cmd, to_text(out)))
+            module.fail_json(msg=u'Failed to decode output from %s: %s' % (item, to_text(item)))
-
-        responses.append(out)
 
+        responses.append(item)
     return responses
 
 
@@ -167,7 +157,6 @@ def load_config(module, commands):
 
     try:
         resp = connection.edit_config(commands)
-        resp = json.loads(resp)
         return resp.get('response')
     except ConnectionError as exc:
         module.fail_json(msg=to_text(exc))

lib/ansible/module_utils/network/vyos/vyos.py

@@ -103,28 +103,21 @@ def run_commands(module, commands, check_rc=True):
     responses = list()
     connection = get_connection(module)
 
-    for cmd in to_list(commands):
+    try:
-        try:
+        outputs = connection.run_commands(commands)
-            cmd = json.loads(cmd)
+    except ConnectionError as exc:
-            command = cmd['command']
+        if check_rc:
-            prompt = cmd['prompt']
-            answer = cmd['answer']
-        except:
-            command = cmd
-            prompt = None
-            answer = None
-
-        try:
-            out = connection.get(command, prompt, answer)
-        except ConnectionError as exc:
             module.fail_json(msg=to_text(exc))
+        else:
+            outputs = exc
 
+    for item in to_list(outputs):
         try:
-            out = to_text(out, errors='surrogate_or_strict')
+            item = to_text(item, errors='surrogate_or_strict')
         except UnicodeError:
-            module.fail_json(msg=u'Failed to decode output from %s: %s' % (cmd, to_text(out)))
+            module.fail_json(msg=u'Failed to decode output from %s: %s' % (item, to_text(item)))
 
-        responses.append(out)
+        responses.append(item)
 
     return responses
 

lib/ansible/modules/network/vyos/vyos_command.py

@@ -168,7 +168,7 @@ def parse_commands(module, warnings):
             warnings.append('only show commands are supported when using '
                             'check mode, not executing `%s`' % item['command'])
         else:
-            items.append(module.jsonify(item))
+            items.append(item)
 
     return items
 

lib/ansible/plugins/cliconf/__init__.py

@@ -163,7 +163,7 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         self.response_logging = False
 
     @abstractmethod
-    def get_config(self, source='running', filter=None, format='text'):
+    def get_config(self, source='running', filter=None, format=None):
         """Retrieves the specified configuration from the device
 
         This method will retrieve the configuration specified by source and
@@ -215,7 +215,7 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         pass
 
     @abstractmethod
-    def get(self, command=None, prompt=None, answer=None, sendonly=False, newline=True):
+    def get(self, command=None, prompt=None, answer=None, sendonly=False, newline=True, output=None):
         """Execute specified command on remote device
         This method will retrieve the specified data and
         return it to the caller as a string.
@@ -225,6 +225,9 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         :param answer: the string to respond to the prompt with
         :param sendonly: bool to disable waiting for response, default is false
         :param newline: bool to indicate if newline should be added at end of answer or not
+        :param output: For devices that support fetching command output in different
+            format, this keyword argument is used to specify the output in which
+            response is to be retrieved.
         :return:
         """
         pass
@@ -264,6 +267,7 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
                 'format': [list of supported configuration format],
                 'diff_match': [list of supported match values],
                 'diff_replace': [list of supported replace values],
+                'output': [list of supported command output format]
             }
         :return: capability as json string
         """
@@ -369,3 +373,22 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
                }
 
         """
+        pass
+
+    def run_commands(self, commands):
+        """
+        Execute a list of commands on remote host and return the list of response
+        :param commands: The list of command that needs to be executed on remote host.
+                The individual command in list can either be a command string or command dict.
+                If the command is dict the valid keys are
+                {
+                    'command': <command to be executed>
+                    'prompt': <expected prompt on executing the command>,
+                    'answer': <answer for the prompt>,
+                    'output': <the format in which command output should be rendered eg: 'json', 'text', if supported by platform>,
+                    'sendonly': <Boolean flag to indicate if it command execution response should be ignored or not>
+                }
+
+        :return: List of returned response
+        """
+        pass

lib/ansible/plugins/cliconf/ios.py

@@ -36,10 +36,13 @@ from ansible.plugins.cliconf import CliconfBase, enable_mode
 class Cliconf(CliconfBase):
 
     @enable_mode
-    def get_config(self, source='running', filter=None, format='text'):
+    def get_config(self, source='running', filter=None, format=None):
         if source not in ('running', 'startup'):
             return self.invalid_params("fetching configuration from %s is not supported" % source)
 
+        if format:
+            raise ValueError("'format' value %s is not supported on ios" % format)
+
         if not filter:
             filter = []
         if source == 'running':
@@ -152,11 +155,14 @@ class Cliconf(CliconfBase):
             results.append(self.send_command('end'))
 
         resp['response'] = results[1:-1]
-        return json.dumps(resp)
+        return resp
 
-    def get(self, command=None, prompt=None, answer=None, sendonly=False):
+    def get(self, command=None, prompt=None, answer=None, sendonly=False, output=None):
         if not command:
             raise ValueError('must provide value of command to execute')
+        if output:
+            raise ValueError("'output' value %s is not supported on ios" % output)
+
         return self.send_command(command=command, prompt=prompt, answer=answer, sendonly=sendonly)
 
     def get_device_info(self):
@@ -199,19 +205,20 @@ class Cliconf(CliconfBase):
         return {
             'format': ['text'],
             'diff_match': ['line', 'strict', 'exact', 'none'],
-            'diff_replace': ['line', 'block']
+            'diff_replace': ['line', 'block'],
+            'output': []
         }
 
     def get_capabilities(self):
         result = dict()
-        result['rpc'] = self.get_base_rpc() + ['edit_banner', 'get_diff']
+        result['rpc'] = self.get_base_rpc() + ['edit_banner', 'get_diff', 'run_commands']
         result['network_api'] = 'cliconf'
         result['device_info'] = self.get_device_info()
         result['device_operations'] = self.get_device_operations()
         result.update(self.get_option_values())
         return json.dumps(result)
 
-    def edit_banner(self, candidate=None, multiline_delimiter="@", commit=True, diff=False):
+    def edit_banner(self, candidate=None, multiline_delimiter="@", commit=True):
         """
         Edit banner on remote device
         :param banners: Banners to be loaded in json format
@@ -223,6 +230,7 @@ class Cliconf(CliconfBase):
         :return: Returns response of executing the configuration command received
              from remote host
         """
+        resp = {}
         banners_obj = json.loads(candidate)
         results = []
         if commit:
@@ -235,11 +243,22 @@ class Cliconf(CliconfBase):
                 time.sleep(0.1)
                 results.append(self.send_command('\n'))
 
-        diff_banner = None
+        resp['response'] = results[1:-1]
-        if diff:
-            diff_banner = candidate
 
-        return diff_banner, results[1:-1]
+        return resp
+
+    def run_commands(self, commands):
+        responses = list()
+        for cmd in to_list(commands):
+            if not isinstance(cmd, collections.Mapping):
+                cmd = {'command': cmd}
+
+            output = cmd.pop('output', None)
+            if output:
+                raise ValueError("'output' value %s is not supported on ios" % output)
+
+            responses.append(self.send_command(**cmd))
+        return responses
 
     def _extract_banners(self, config):
         banners = {}

lib/ansible/plugins/cliconf/vyos.py

@@ -54,7 +54,12 @@ class Cliconf(CliconfBase):
 
         return device_info
 
-    def get_config(self, filter=None, format='set'):
+    def get_config(self, filter=None, format=None):
+        if format:
+            option_values = self.get_option_values()
+            if format not in option_values['format']:
+                raise ValueError("'format' value %s is invalid. Valid values of format are %s" % (format, ','.join(option_values['format'])))
+
         if format == 'text':
             out = self.send_command('show configuration')
         else:
@@ -97,19 +102,23 @@ class Cliconf(CliconfBase):
                     self.discard_changes()
                     raise AnsibleConnectionFailure(msg)
                 else:
-                    self.get('exit')
+                    self.send_command('exit')
             else:
                 self.discard_changes()
         else:
-            self.get('exit')
+            self.send_command('exit')
 
         resp['diff'] = diff_config
         resp['response'] = results[1:-1]
         return json.dumps(resp)
 
-    def get(self, command=None, prompt=None, answer=None, sendonly=False):
+    def get(self, command=None, prompt=None, answer=None, sendonly=False, output=None):
         if not command:
             raise ValueError('must provide value of command to execute')
+
+        if output:
+            raise ValueError("'output' value %s is not supported on vyos" % output)
+
         return self.send_command(command, prompt=prompt, answer=answer, sendonly=sendonly)
 
     def commit(self, comment=None):
@@ -191,6 +200,19 @@ class Cliconf(CliconfBase):
         diff['config_diff'] = list(updates)
         return json.dumps(diff)
 
+    def run_commands(self, commands):
+        responses = list()
+        for cmd in to_list(commands):
+            if not isinstance(cmd, collections.Mapping):
+                cmd = {'command': cmd}
+
+            output = cmd.pop('output', None)
+            if output:
+                raise ValueError("'output' value %s is not supported on vyos" % output)
+
+            responses.append(self.send_command(**cmd))
+        return responses
+
     def get_device_operations(self):
         return {
             'supports_diff_replace': False,
@@ -208,14 +230,15 @@ class Cliconf(CliconfBase):
 
     def get_option_values(self):
         return {
-            'format': ['set', 'text'],
+            'format': ['text', 'set'],
             'diff_match': ['line', 'none'],
             'diff_replace': [],
+            'output': []
         }
 
     def get_capabilities(self):
         result = {}
-        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'get_diff']
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'get_diff', 'run_commands']
         result['network_api'] = 'cliconf'
         result['device_info'] = self.get_device_info()
         result['device_operations'] = self.get_device_operations()

test/units/modules/network/vyos/test_vyos_command.py

@@ -47,8 +47,7 @@ class TestVyosCommandModule(TestVyosModule):
 
             for item in commands:
                 try:
-                    obj = json.loads(item)
+                    command = item['command']
-                    command = obj['command']
                 except ValueError:
                     command = item
                 filename = str(command).replace(' ', '_')