From 387a23c3d14f6c29ef6e78ca333c6a054eac5730 Mon Sep 17 00:00:00 2001 From: wiso Date: Thu, 24 May 2018 11:55:02 +0200 Subject: [PATCH] New ansible module netconf_rpc (#40358) * New ansible module netconf_rpc * add integration test for module netconf_rpc * pep8/meta-data corrections * usage of jxmlease for all XML processing separation of attributes "rpc" and "content" * removed unused imports improved error handling * fixed pep8 * usage of ast.literal_eval instead of eval added description to SROS integration test for cases commented out --- .github/BOTMETA.yml | 1 + .../module_utils/network/netconf/netconf.py | 18 +- .../modules/network/netconf/netconf_rpc.py | 262 ++++++++++++++++++ lib/ansible/plugins/netconf/__init__.py | 37 ++- .../targets/netconf_rpc/defaults/main.yaml | 2 + .../targets/netconf_rpc/meta/main.yml | 4 + .../targets/netconf_rpc/tasks/iosxr.yaml | 16 ++ .../targets/netconf_rpc/tasks/junos.yaml | 16 ++ .../targets/netconf_rpc/tasks/main.yaml | 4 + .../targets/netconf_rpc/tasks/sros.yaml | 16 ++ .../netconf_rpc/tests/iosxr/basic.yaml | 8 + .../netconf_rpc/tests/junos/basic.yaml | 8 + .../targets/netconf_rpc/tests/sros/basic.yaml | 188 +++++++++++++ 13 files changed, 561 insertions(+), 19 deletions(-) create mode 100644 lib/ansible/modules/network/netconf/netconf_rpc.py create mode 100644 test/integration/targets/netconf_rpc/defaults/main.yaml create mode 100644 test/integration/targets/netconf_rpc/meta/main.yml create mode 100644 test/integration/targets/netconf_rpc/tasks/iosxr.yaml create mode 100644 test/integration/targets/netconf_rpc/tasks/junos.yaml create mode 100644 test/integration/targets/netconf_rpc/tasks/main.yaml create mode 100644 test/integration/targets/netconf_rpc/tasks/sros.yaml create mode 100644 test/integration/targets/netconf_rpc/tests/iosxr/basic.yaml create mode 100644 test/integration/targets/netconf_rpc/tests/junos/basic.yaml create mode 100644 test/integration/targets/netconf_rpc/tests/sros/basic.yaml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index adb230157a..bafbcdc9e1 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -481,6 +481,7 @@ files: $modules/network/meraki/: $team_meraki $modules/network/netconf/netconf_config.py: lpenz userlerueda $team_networking $modules/network/netconf/netconf_get.py: wisotzky $team_networking + $modules/network/netconf/netconf_rpc.py: wisotzky $team_networking $modules/network/netscaler/: $team_netscaler $modules/network/netvisor/: $team_netvisor $modules/network/nuage/: pdellaert diff --git a/lib/ansible/module_utils/network/netconf/netconf.py b/lib/ansible/module_utils/network/netconf/netconf.py index d946ec45b3..9abb5d7c98 100644 --- a/lib/ansible/module_utils/network/netconf/netconf.py +++ b/lib/ansible/module_utils/network/netconf/netconf.py @@ -48,13 +48,13 @@ def get_capabilities(module): return module._netconf_capabilities -def lock_configuration(x, target=None): - conn = get_connection(x) +def lock_configuration(module, target=None): + conn = get_connection(module) return conn.lock(target=target) -def unlock_configuration(x, target=None): - conn = get_connection(x) +def unlock_configuration(module, target=None): + conn = get_connection(module) return conn.unlock(target=target) @@ -104,3 +104,13 @@ def get(module, filter, lock=False): conn.unlock(target='running') return response + + +def dispatch(module, request): + conn = get_connection(module) + try: + response = conn.dispatch(request) + except ConnectionError as e: + module.fail_json(msg=to_text(e, errors='surrogate_then_replace').strip()) + + return response diff --git a/lib/ansible/modules/network/netconf/netconf_rpc.py b/lib/ansible/modules/network/netconf/netconf_rpc.py new file mode 100644 index 0000000000..b1e013b3ab --- /dev/null +++ b/lib/ansible/modules/network/netconf/netconf_rpc.py @@ -0,0 +1,262 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2018, Ansible by Red Hat, inc +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'network'} + + +DOCUMENTATION = """ +--- +module: netconf_rpc +version_added: "2.6" +author: + - "Ganesh Nalawade (@ganeshrn)" + - "Sven Wisotzky (@wisotzky)" +short_description: Execute operations on NETCONF enabled network devices. +description: + - NETCONF is a network management protocol developed and standardized by + the IETF. It is documented in RFC 6241. + - This module allows the user to execute NETCONF RPC requests as defined + by IETF RFC standards as well as proprietary requests. +options: + rpc: + description: + - This argument specifies the request (name of the operation) to be executed on + the remote NETCONF enabled device. + xmlns: + description: + - NETCONF operations not defined in rfc6241 typically require the appropriate + XML namespace to be set. In the case the I(request) option is not already + provided in XML format, the namespace can be defined by the I(xmlns) + option. + content: + description: + - This argument specifies the optional request content (all RPC attributes). + The I(content) value can either be provided as XML formatted string or as + dictionary. + display: + description: + - Encoding scheme to use when serializing output from the device. The option I(json) will + serialize the output as JSON data. If the option value is I(json) it requires jxmlease + to be installed on control node. The option I(pretty) is similar to received XML response + but is using human readable format (spaces, new lines). The option value I(xml) is similar + to received XML response but removes all XML namespaces. + choices: ['json', 'pretty', 'xml'] +requirements: + - ncclient (>=v0.5.2) + - jxmlease + +notes: + - This module requires the NETCONF system service be enabled on the remote device + being managed. + - This module supports the use of connection=netconf + - To execute C(get-config), C(get) or C(edit-config) requests it is recommended + to use the Ansible I(netconf_get) and I(netconf_config) modules. +""" + +EXAMPLES = """ +- name: lock candidate + netconf_rpc: + rpc: lock + content: + target: + candidate: + +- name: unlock candidate + netconf_rpc: + rpc: unlock + xmlns: "urn:ietf:params:xml:ns:netconf:base:1.0" + content: "{'target': {'candidate': None}}" + +- name: discard changes + netconf_rpc: + rpc: discard-changes + +- name: get-schema + netconf_rpc: + rpc: get-schema + xmlns: urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring + content: + identifier: ietf-netconf + version: "2011-06-01" + +- name: copy running to startup + netconf_rpc: + rpc: copy-config + content: + source: + running: + target: + startup: + +- name: get schema list with JSON output + netconf_rpc: + rpc: get + content: | + + + + + + display: json + +- name: get schema using XML request + netconf_rpc: + rpc: "get-schema" + xmlns: "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring" + content: | + ietf-netconf-monitoring + 2010-10-04 + display: json +""" + +RETURN = """ +stdout: + description: The raw XML string containing configuration or state data + received from the underlying ncclient library. + returned: always apart from low-level errors (such as action plugin) + type: string + sample: '...' +stdout_lines: + description: The value of stdout split into a list + returned: always apart from low-level errors (such as action plugin) + type: list + sample: ['...', '...'] +output: + description: Based on the value of display option will return either the set of + transformed XML to JSON format from the RPC response with type dict + or pretty XML string response (human-readable) or response with + namespace removed from XML string. + returned: when the display format is selected as JSON it is returned as dict type, if the + display format is xml or pretty pretty it is retured as a string apart from low-level + errors (such as action plugin). + type: complex + contains: + formatted_output: + - Contains formatted response received from remote host as per the value in display format. +""" + +import ast + +try: + from lxml.etree import tostring +except ImportError: + from xml.etree.ElementTree import tostring + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.network.netconf.netconf import dispatch +from ansible.module_utils.network.common.netconf import remove_namespaces + +try: + import jxmlease + HAS_JXMLEASE = True +except ImportError: + HAS_JXMLEASE = False + + +def get_xml_request(module, request, xmlns, content): + if content is None: + if xmlns is None: + return '<%s/>' % request + else: + return '<%s xmlns="%s"/>' % (request, xmlns) + + if isinstance(content, str): + content = content.strip() + + if content.startswith('<') and content.endswith('>'): + # assumption content contains already XML payload + if xmlns is None: + return '<%s>%s' % (request, content, request) + else: + return '<%s xmlns="%s">%s' % (request, xmlns, content, request) + + try: + # trying if content contains dict + content = ast.literal_eval(content) + except: + module.fail_json(msg='unsupported content value `%s`' % content) + + if isinstance(content, dict): + if not HAS_JXMLEASE: + module.fail_json(msg='jxmlease is required to convert RPC content to XML ' + 'but does not appear to be installed. ' + 'It can be installed using `pip install jxmlease`') + + payload = jxmlease.XMLDictNode(content).emit_xml(pretty=False, full_document=False) + if xmlns is None: + return '<%s>%s' % (request, payload, request) + else: + return '<%s xmlns="%s">%s' % (request, xmlns, payload, request) + + module.fail_json(msg='unsupported content data-type `%s`' % type(content).__name__) + + +def main(): + """entry point for module execution + """ + argument_spec = dict( + rpc=dict(type="str", required=True), + xmlns=dict(type="str"), + content=dict(), + display=dict(choices=['json', 'pretty', 'xml']) + ) + + module = AnsibleModule(argument_spec=argument_spec, + supports_check_mode=True) + + rpc = module.params['rpc'] + xmlns = module.params['xmlns'] + content = module.params['content'] + display = module.params['display'] + + if rpc is None: + module.fail_json(msg='argument `rpc` must not be None') + + rpc = rpc.strip() + if len(rpc) == 0: + module.fail_json(msg='argument `rpc` must not be empty') + + if rpc in ['close-session']: + # explicit close-session is not allowed, as this would make the next + # NETCONF operation to the same host fail + module.fail_json(msg='unsupported operation `%s`' % rpc) + + if display == 'json' and not HAS_JXMLEASE: + module.fail_json(msg='jxmlease is required to display response in json format' + 'but does not appear to be installed. ' + 'It can be installed using `pip install jxmlease`') + + xml_req = get_xml_request(module, rpc, xmlns, content) + response = dispatch(module, xml_req) + + xml_resp = tostring(response) + output = None + + if display == 'xml': + output = remove_namespaces(xml_resp) + elif display == 'json': + try: + output = jxmlease.parse(xml_resp) + except: + raise ValueError(xml_resp) + elif display == 'pretty': + output = tostring(response, pretty_print=True) + + result = { + 'stdout': xml_resp, + 'output': output + } + + module.exit_json(**result) + +if __name__ == '__main__': + main() diff --git a/lib/ansible/plugins/netconf/__init__.py b/lib/ansible/plugins/netconf/__init__.py index 8dd30e788e..c193d47bf3 100644 --- a/lib/ansible/plugins/netconf/__init__.py +++ b/lib/ansible/plugins/netconf/__init__.py @@ -32,6 +32,11 @@ try: except ImportError: raise AnsibleError("ncclient is not installed") +try: + from lxml.etree import Element, SubElement, tostring, fromstring +except ImportError: + from xml.etree.ElementTree import Element, SubElement, tostring, fromstring + def ensure_connected(func): @wraps(func) @@ -106,7 +111,7 @@ class NetconfBase(with_metaclass(ABCMeta, object)): resp = self.m.rpc(obj) return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml except RPCError as exc: - msg = exc.data_xml if hasattr(exc, 'data_xml') else exc.xml + msg = exc.xml raise Exception(to_xml(msg)) @ensure_connected @@ -174,6 +179,15 @@ class NetconfBase(with_metaclass(ABCMeta, object)): resp = self.m.copy_config(*args, **kwargs) return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + @ensure_connected + def dispatch(self, request): + """Execute operation on the remote device + :request: is the rpc request including attributes as XML string + """ + req = fromstring(request) + resp = self.m.dispatch(req) + return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml + @ensure_connected def lock(self, target=None): """ @@ -228,13 +242,6 @@ class NetconfBase(with_metaclass(ABCMeta, object)): resp = self.m.commit(*args, **kwargs) return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml - @ensure_connected - def validate(self, *args, **kwargs): - """Validate the contents of the specified configuration. - :source: name of configuration data store""" - resp = self.m.validate(*args, **kwargs) - return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml - @ensure_connected def get_schema(self, *args, **kwargs): """Retrieves the required schema from the device @@ -277,15 +284,15 @@ class NetconfBase(with_metaclass(ABCMeta, object)): def get_device_operations(self, server_capabilities): operations = {} capabilities = '\n'.join(server_capabilities) - operations['supports_commit'] = True if ':candidate' in capabilities else False - operations['supports_defaults'] = True if ':with-defaults' in capabilities else False - operations['supports_confirm_commit'] = True if ':confirmed-commit' in capabilities else False - operations['supports_startup'] = True if ':startup' in capabilities else False - operations['supports_xpath'] = True if ':xpath' in capabilities else False - operations['supports_writeable_running'] = True if ':writable-running' in capabilities else False + operations['supports_commit'] = ':candidate' in capabilities + operations['supports_defaults'] = ':with-defaults' in capabilities + operations['supports_confirm_commit'] = ':confirmed-commit' in capabilities + operations['supports_startup'] = ':startup' in capabilities + operations['supports_xpath'] = ':xpath' in capabilities + operations['supports_writable_running'] = ':writable-running' in capabilities operations['lock_datastore'] = [] - if operations['supports_writeable_running']: + if operations['supports_writable_running']: operations['lock_datastore'].append('running') if operations['supports_commit']: diff --git a/test/integration/targets/netconf_rpc/defaults/main.yaml b/test/integration/targets/netconf_rpc/defaults/main.yaml new file mode 100644 index 0000000000..5f709c5aac --- /dev/null +++ b/test/integration/targets/netconf_rpc/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +testcase: "*" diff --git a/test/integration/targets/netconf_rpc/meta/main.yml b/test/integration/targets/netconf_rpc/meta/main.yml new file mode 100644 index 0000000000..3403f48112 --- /dev/null +++ b/test/integration/targets/netconf_rpc/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - { role: prepare_junos_tests, when: ansible_network_os == 'junos' } + - { role: prepare_iosxr_tests, when: ansible_network_os == 'iosxr' } diff --git a/test/integration/targets/netconf_rpc/tasks/iosxr.yaml b/test/integration/targets/netconf_rpc/tasks/iosxr.yaml new file mode 100644 index 0000000000..7894985531 --- /dev/null +++ b/test/integration/targets/netconf_rpc/tasks/iosxr.yaml @@ -0,0 +1,16 @@ +--- +- name: collect all netconf test cases + find: + paths: "{{ role_path }}/tests/iosxr" + patterns: "{{ testcase }}.yaml" + register: test_cases + connection: local + +- name: set test_items + set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" + +- name: run test case (connection=netconf) + include: "{{ test_case_to_run }} ansible_connection=netconf" + with_items: "{{ test_items }}" + loop_control: + loop_var: test_case_to_run diff --git a/test/integration/targets/netconf_rpc/tasks/junos.yaml b/test/integration/targets/netconf_rpc/tasks/junos.yaml new file mode 100644 index 0000000000..86c56f83a5 --- /dev/null +++ b/test/integration/targets/netconf_rpc/tasks/junos.yaml @@ -0,0 +1,16 @@ +--- +- name: collect all netconf test cases + find: + paths: "{{ role_path }}/tests/junos" + patterns: "{{ testcase }}.yaml" + register: test_cases + connection: local + +- name: set test_items + set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" + +- name: run test case (connection=netconf) + include: "{{ test_case_to_run }} ansible_connection=netconf" + with_items: "{{ test_items }}" + loop_control: + loop_var: test_case_to_run diff --git a/test/integration/targets/netconf_rpc/tasks/main.yaml b/test/integration/targets/netconf_rpc/tasks/main.yaml new file mode 100644 index 0000000000..a34a2fecd6 --- /dev/null +++ b/test/integration/targets/netconf_rpc/tasks/main.yaml @@ -0,0 +1,4 @@ +--- +- { include: junos.yaml, when: ansible_network_os == 'junos', tags: ['netconf'] } +- { include: iosxr.yaml, when: ansible_network_os == 'iosxr', tags: ['netconf'] } +- { include: sros.yaml, when: ansible_network_os == 'sros', tags: ['netconf'] } diff --git a/test/integration/targets/netconf_rpc/tasks/sros.yaml b/test/integration/targets/netconf_rpc/tasks/sros.yaml new file mode 100644 index 0000000000..bc8728b82e --- /dev/null +++ b/test/integration/targets/netconf_rpc/tasks/sros.yaml @@ -0,0 +1,16 @@ +--- +- name: collect all netconf test cases + find: + paths: "{{ role_path }}/tests/sros" + patterns: "{{ testcase }}.yaml" + register: test_cases + connection: local + +- name: set test_items + set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" + +- name: run test case (connection=netconf) + include: "{{ test_case_to_run }} ansible_connection=netconf" + with_items: "{{ test_items }}" + loop_control: + loop_var: test_case_to_run diff --git a/test/integration/targets/netconf_rpc/tests/iosxr/basic.yaml b/test/integration/targets/netconf_rpc/tests/iosxr/basic.yaml new file mode 100644 index 0000000000..992d051692 --- /dev/null +++ b/test/integration/targets/netconf_rpc/tests/iosxr/basic.yaml @@ -0,0 +1,8 @@ +--- +- debug: msg="START netconf_rpc iosxr/basic.yaml on connection={{ ansible_connection }}" + +- name: discard changes + netconf_rpc: + rpc: discard-changes + +- debug: msg="END netconf_rpc iosxr/basic.yaml on connection={{ ansible_connection }}" diff --git a/test/integration/targets/netconf_rpc/tests/junos/basic.yaml b/test/integration/targets/netconf_rpc/tests/junos/basic.yaml new file mode 100644 index 0000000000..956a1e424d --- /dev/null +++ b/test/integration/targets/netconf_rpc/tests/junos/basic.yaml @@ -0,0 +1,8 @@ +--- +- debug: msg="START netconf_rpc junos/basic.yaml on connection={{ ansible_connection }}" + +- name: discard changes + netconf_rpc: + rpc: discard-changes + +- debug: msg="END netconf_rpc junos/basic.yaml on connection={{ ansible_connection }}" diff --git a/test/integration/targets/netconf_rpc/tests/sros/basic.yaml b/test/integration/targets/netconf_rpc/tests/sros/basic.yaml new file mode 100644 index 0000000000..f7e58a3651 --- /dev/null +++ b/test/integration/targets/netconf_rpc/tests/sros/basic.yaml @@ -0,0 +1,188 @@ +--- +- debug: msg="START netconf_rpc sros/basic.yaml on connection={{ ansible_connection }}" + +- name: lock candidate (content is dict) + netconf_rpc: + rpc: lock + content: + target: + candidate: + register: result + connection: netconf + +- name: discard changes (w/o content) + netconf_rpc: + rpc: discard-changes + display: xml + register: result + connection: netconf + +- name: unlock candidate (content is dict as json) + netconf_rpc: + rpc: unlock + xmlns: "urn:ietf:params:xml:ns:netconf:base:1.0" + content: "{'target': {'candidate': None}}" + display: json + register: result + connection: netconf + +- assert: + that: + - "{{ result['output']['rpc-reply'] is defined}}" + - "{{ result['output']['rpc-reply']['ok'] is defined}}" + +- name: validate candidate (content is single line of XML) + netconf_rpc: + rpc: validate + content: "" + display: json + register: result + connection: netconf + +- assert: + that: + - "{{ result['output']['rpc-reply'] is defined}}" + - "{{ result['output']['rpc-reply']['ok'] is defined}}" + +- name: copy running to startup + netconf_rpc: + rpc: copy-config + content: + source: + running: + target: + startup: + register: result + connection: netconf + +- name: get schema list (content is multiple lines of XML) + netconf_rpc: + rpc: get + content: | + + + + + + display: json + register: result + connection: netconf + +- assert: + that: + - "{{ result['output']['data'] is defined}}" + - "{{ result['output']['data']['netconf-state'] is defined}}" + - "{{ result['output']['data']['netconf-state']['schemas'] is defined}}" + - "{{ result['output']['data']['netconf-state']['schemas']['schema'] is defined}}" + +# The following two test-cases have been validated against a pre-release implementation. +# To make this playbook work with the regular Nokia SROS 16.0 release, those test-cases +# have been commented out. As soon the operation is supported by SROS +# those test-cases shall be included. + +#- name: get-schema +# netconf_rpc: +# rpc: get-schema +# xmlns: urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring +# content: +# identifier: ietf-netconf +# version: "2011-06-01" +# register: result +# connection: netconf + +#- name: get schema using XML request +# netconf_rpc: +# rpc: "get-schema" +# xmlns: "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring" +# content: | +# ietf-netconf-monitoring +# 2010-10-04 +# display: pretty +# register: result +# connection: netconf + +- name: Failure scenario, unsupported content (xpath value) + netconf_rpc: + rpc: get + content: schemas/schema[identifier=ietf-netconf-monitoring] + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'unsupported content value' in result.msg" + +- name: Failure scenario, unsupported content type (list) + netconf_rpc: + rpc: get + content: + - value1 + - value2 + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'unsupported content data-type' in result.msg" + +- name: Failure scenario, RPC is close-session + netconf_rpc: + rpc: close-session + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'unsupported operation' in result.msg" + +- name: Failure scenario, attribute rpc missing + netconf_rpc: + display: json + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'missing required arguments' in result.msg" + +- name: Failure scenario, attribute rpc is None + netconf_rpc: + rpc: + display: json + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'must not be None' in result.msg" + +- name: Failure scenario, attribute rpc is zero-length string + netconf_rpc: + rpc: "" + display: json + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'must not be empty' in result.msg" + +- name: Failure scenario, attribute rpc only contains white-spaces + netconf_rpc: + rpc: " " + display: json + register: result + connection: netconf + ignore_errors: True + +- assert: + that: + - "'must not be empty' in result.msg" + +- debug: msg="END netconf_rpc sros/basic.yaml on connection={{ ansible_connection }}"