[WIP] Add new network module for EdgeOS (#33109)

* Add edgeos network module using network_cli

* Add documentation and some sanity test fixes

* Update copyright and documentation

lib/ansible/module_utils/network/edgeos/edgeos.py

@@ -0,0 +1,126 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# (c) 2016 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+import json
+from ansible.module_utils._text import to_text
+from ansible.module_utils.network.common.utils import to_list
+from ansible.module_utils.connection import Connection
+
+_DEVICE_CONFIGS = None
+
+
+def get_connection(module):
+    if hasattr(module, '_edgeos_connection'):
+        return module._edgeos_connection
+
+    capabilities = get_capabilities(module)
+    network_api = capabilities.get('network_api')
+    if network_api == 'cliconf':
+        module._edgeos_connection = Connection(module._socket_path)
+    else:
+        module.fail_json(msg='Invalid connection type %s' % network_api)
+
+    return module._edgeos_connection
+
+
+def get_capabilities(module):
+    if hasattr(module, '_edgeos_capabilities'):
+        return module._edgeos_capabilities
+
+    capabilities = Connection(module._socket_path).get_capabilities()
+    module._edgeos_capabilities = json.loads(capabilities)
+    return module._edgeos_capabilities
+
+
+def get_config(module):
+    global _DEVICE_CONFIGS
+
+    if _DEVICE_CONFIGS is not None:
+        return _DEVICE_CONFIGS
+    else:
+        connection = get_connection(module)
+        out = connection.get_config()
+        cfg = to_text(out, errors='surrogate_then_replace').strip()
+        _DEVICE_CONFIGS = cfg
+        return cfg
+
+
+def run_commands(module, commands, check_rc=True):
+    responses = list()
+    connection = get_connection(module)
+
+    for cmd in to_list(commands):
+        if isinstance(cmd, dict):
+            command = cmd['command']
+            prompt = cmd['prompt']
+            answer = cmd['answer']
+        else:
+            command = cmd
+            prompt = None
+            answer = None
+
+        out = connection.get(command, prompt, answer)
+
+        try:
+            out = to_text(out, errors='surrogate_or_strict')
+        except UnicodeError:
+            module.fail_json(msg=u'Failed to decode output from %s: %s' %
+                             (cmd, to_text(out)))
+
+        responses.append(out)
+
+    return responses
+
+
+def load_config(module, commands, commit=False, comment=None):
+    connection = get_connection(module)
+
+    out = connection.edit_config(commands)
+
+    diff = None
+    if module._diff:
+        out = connection.get('compare')
+        out = to_text(out, errors='surrogate_or_strict')
+
+        if not out.startswith('No changes'):
+            out = connection.get('show')
+            diff = to_text(out, errors='surrogate_or_strict').strip()
+
+    if commit:
+        try:
+            out = connection.commit(comment)
+        except:
+            connection.discard_changes()
+            module.fail_json(msg='commit failed: %s' % out)
+
+    if not commit:
+        connection.discard_changes()
+    else:
+        connection.get('exit')
+
+    if diff:
+        return diff

lib/ansible/modules/network/edgeos/edgeos_command.py

@@ -0,0 +1,190 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2017, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+DOCUMENTATION = """
+module: edgeos_command
+version_added: "2.5"
+author:
+  - Chad Norgan (@BeardyMcBeards)
+  - Sam Doran (@samdoran)
+short_description: Run one or more commands on EdgeOS devices
+description:
+  - This command module allows running one or more commands on a remote
+    device running EdgeOS, such as the Ubiquiti EdgeRouter.
+  - This module does not support running commands in configuration mode.
+  - Certain C(show) commands in EdgeOS produce many lines of output and
+    use a custom pager that can cause this module to hang.  If the
+    value of the environment variable C(ANSIBLE_EDGEOS_TERMINAL_LENGTH)
+    is not set, the default number of 10000 is used.
+options:
+  commands:
+    description:
+      - The commands or ordered set of commands that should be run against the
+        remote device. The output of the command is returned to the playbook.
+        If the C(wait_for) argument is provided, the module is not returned
+        until the condition is met or the number of retries is exceeded.
+    required: True
+  wait_for:
+    description:
+      - Causes the task to wait for a specific condition to be met before
+        moving forward. If the condition is not met before the specified
+        number of retries is exceeded, the task will fail.
+    required: False
+  match:
+    description:
+      - Used in conjunction with C(wait_for) to create match policy. If set to
+        C(all), then all conditions in C(wait_for) must be met. If set to
+        C(any), then only one condition must match.
+    required: False
+    default: 'all'
+    choices: ['any', 'all']
+  retries:
+    description:
+      - Number of times a command should be tried before it is considered failed.
+        The command is run on the target device and evaluated against the
+        C(wait_for) conditionals.
+    required: False
+    default: 10
+  interval:
+    description:
+      - The number of seconds to wait between C(retries) of the command.
+    required: False
+    default: 1
+
+notes:
+  - Tested against EdgeOS 1.9.7
+  - Running C(show system boot-messages all) will cause the module to hang since
+    EdgeOS is using a custom pager setting to display the output of that command.
+"""
+
+EXAMPLES = """
+tasks:
+  - name: Reboot the device
+    edgeos_command:
+      commands: reboot now
+
+  - name: Show the configuration for eth0 and eth1
+    edgeos_command:
+      commands: show interfaces ethernet {{ item }}
+    loop:
+      - eth0
+      - eth1
+"""
+
+RETURN = """
+stdout:
+  description: The set of responses from the commands
+  returned: always apart from low level errors (such as action plugin)
+  type: list
+  sample: ['...', '...']
+stdout_lines:
+  description: The value of stdout split into a list
+  returned: always
+  type: list
+  sample: [['...', '...'], ['...'], ['...']]
+"""
+
+import time
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.common.parsing import Conditional
+from ansible.module_utils.network.edgeos.edgeos import run_commands
+from ansible.module_utils.six import string_types
+
+
+def to_lines(stdout):
+    for item in stdout:
+        if isinstance(item, string_types):
+            item = str(item).split('\n')
+        yield item
+
+
+def parse_commands(module, warnings):
+    spec = dict(
+        command=dict(key=True),
+        prompt=dict(),
+        answer=dict(),
+    )
+
+    transform = ComplexList(spec, module)
+    commands = transform(module.params['commands'])
+
+    if module.check_mode:
+        for item in list(commands):
+            if not item['command'].startswith('show'):
+                warnings.append(
+                    'Only show commands are supported when using check_mode, '
+                    'not executing %s' % item['command'])
+                commands.remove(item)
+
+    return commands
+
+
+def main():
+    spec = dict(
+        commands=dict(type='list', required=True),
+        wait_for=dict(type='list'),
+        match=dict(default='all', choices=['all', 'any']),
+        retries=dict(default=10, type='int'),
+        interval=dict(default=1, type='int')
+    )
+
+    module = AnsibleModule(argument_spec=spec, supports_check_mode=True)
+
+    warnings = list()
+    result = {'changed': False}
+    commands = parse_commands(module, warnings)
+    wait_for = module.params['wait_for'] or list()
+
+    try:
+        conditionals = [Conditional(c) for c in wait_for]
+    except AttributeError as e:
+        module.fail_json(msg=str(e))
+
+    retries = module.params['retries']
+    interval = module.params['interval']
+    match = module.params['match']
+
+    while retries > 0:
+        responses = run_commands(module, commands)
+
+        for item in list(conditionals):
+            if item(responses):
+                if match == 'any':
+                    conditionals = list()
+                    break
+                conditionals.remove(item)
+
+        if not conditionals:
+            break
+
+        time.sleep(interval)
+        retries -= 1
+
+    if conditionals:
+        failed_conditions = [item.raw for item in conditionals]
+        msg = 'One or more conditional statements have not been satisfied'
+        module.fail_json(msg=msg, falied_conditions=failed_conditions)
+
+    result = {
+        'changed': False,
+        'stdout': responses,
+        'warnings': warnings,
+        'stdout_lines': list(to_lines(responses))
+    }
+
+    module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/plugins/cliconf/edgeos.py

@@ -0,0 +1,69 @@
+# Copyright: (c) 2017, Ansible Project
+# GNU General Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.network.common.utils import to_list
+from ansible.plugins.cliconf import CliconfBase
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'edgeos'
+        reply = self.get(b'show version')
+        data = to_text(reply, errors='surrogate_or_strict').strip()
+
+        match = re.search(r'Version:\s*v?(\S+)', data)
+        if match:
+            device_info['network_os_version'] = match.group(1)
+
+        match = re.search(r'HW model:\s*(\S+)', data)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        reply = self.get(b'show host name')
+        reply = to_text(reply, errors='surrogate_or_strict').strip()
+        device_info['network_os_hostname'] = reply
+
+        return device_info
+
+    def get_config(self):
+        return self.send_command(b'show configuration commands')
+
+    def edit_config(self, command):
+        for cmd in chain([b'configure'], to_list(command)):
+            self.send_command(cmd)
+
+    def get(self, command, prompt=None, answer=None, sendonly=False):
+        return self.send_command(to_bytes(command),
+                                 prompt=to_bytes(prompt),
+                                 answer=to_bytes(answer),
+                                 sendonly=sendonly)
+
+    def commit(self, comment=None):
+        if comment:
+            command = 'commit comment "{0}"'.format(comment)
+        else:
+            command = 'commit'
+        self.send_command(command)
+
+    def discard_changes(self, *args, **kwargs):
+        self.send_command(b'discard')
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes']
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/terminal/edgeos.py

@@ -0,0 +1,36 @@
+# Copyright: (c) 2017, Ansible Project
+# GNU General Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import re
+
+from ansible.plugins.terminal import TerminalBase
+from ansible.errors import AnsibleConnectionFailure
+
+
+class TerminalModule(TerminalBase):
+
+    terminal_stdout_re = [
+        re.compile(br"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
+        re.compile(br"\@[\w\-\.]+:\S+?[>#\$] ?$")
+    ]
+
+    terminal_stderr_re = [
+        re.compile(br"\n\s*command not found"),
+        re.compile(br"\nInvalid command"),
+        re.compile(br"\nCommit failed"),
+        re.compile(br"\n\s*Set failed"),
+    ]
+
+    terminal_length = os.getenv('ANSIBLE_EDGEOS_TERMINAL_LENGTH', 10000)
+
+    def on_open_shell(self):
+        try:
+            self._exec_cli_command('export VYATTA_PAGER=cat')
+            self._exec_cli_command('stty rows %s' % self.terminal_length)
+        except AnsibleConnectionFailure:
+            raise AnsibleConnectionFailure('unable to set terminal parameters')

test/units/modules/network/edgeos/edgeos_module.py

@@ -0,0 +1,86 @@
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import json
+
+from units.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase
+
+
+fixture_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+fixture_data = {}
+
+
+def load_fixture(name):
+    path = os.path.join(fixture_path, name)
+
+    if path in fixture_data:
+        return fixture_data[path]
+
+    with open(path) as f:
+        data = f.read()
+
+    try:
+        data = json.loads(data)
+    except:
+        pass
+
+    fixture_data[path] = data
+    return data
+
+
+class TestEdgeosModule(ModuleTestCase):
+
+    def execute_module(self, failed=False, changed=False, commands=None, sort=True, defaults=False):
+        self.load_fixtures(commands)
+
+        if failed:
+            result = self.failed()
+            self.assertTrue(result['failed'], result)
+        else:
+            result = self.changed(changed)
+            self.assertEqual(result['changed'], changed, result)
+
+        if commands is not None:
+            if sort:
+                self.assertEqual(sorted(commands), sorted(result['commands']), result['commands'])
+            else:
+                self.assertEqual(commands, result['commands'], result['commands'])
+
+        return result
+
+    def failed(self):
+        with self.assertRaises(AnsibleFailJson) as exc:
+            self.module.main()
+
+        result = exc.exception.args[0]
+        self.assertTrue(result['failed'], result)
+        return result
+
+    def changed(self, changed=False):
+        with self.assertRaises(AnsibleExitJson) as exc:
+            self.module.main()
+
+        result = exc.exception.args[0]
+        self.assertEqual(result['changed'], changed, result)
+        return result
+
+    def load_fixtures(self, commands=None):
+        pass

test/units/modules/network/edgeos/fixtures/show_host_name

@@ -0,0 +1 @@
+er01

test/units/modules/network/edgeos/fixtures/show_version

@@ -0,0 +1,7 @@
+Version:      v1.9.7+hotfix.4
+Build ID:     5024004
+Build on:     10/05/17 04:03
+Copyright:    2012-2017 Ubiquiti Networks, Inc.
+HW model:     EdgeRouter PoE 5-Port
+HW S/N:       802AA84D6394
+Uptime:       09:39:34 up 56 days, 21 min,  2 users,  load average: 0.14, 0.11, 0.07

test/units/modules/network/edgeos/test_edgeos_command.py

@@ -0,0 +1,36 @@
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import json
+
+from ansible.compat.tests.mock import patch
+from ansible.modules.network.edgeos import edgeos_command
+from units.modules.utils import set_module_args
+from .edgeos_module import TestEdgeosModule, load_fixture
+
+
+class TestEdgeosCommandModule(TestEdgeosModule):
+
+    module = edgeos_command
+
+    def setUp(self):
+        super(TestEdgeosCommandModule, self).setUp()
+        self.mock_run_commands = patch()