New ansible module netconf_rpc (#40358)

* New ansible module netconf_rpc

* add integration test for module netconf_rpc

* pep8/meta-data corrections

* usage of jxmlease for all XML processing
separation of attributes "rpc" and "content"

* removed unused imports
improved error handling

* fixed pep8

* usage of ast.literal_eval instead of eval
added description to SROS integration test for cases commented out

.github/BOTMETA.yml

@@ -481,6 +481,7 @@ files:
   $modules/network/meraki/: $team_meraki
   $modules/network/netconf/netconf_config.py: lpenz userlerueda $team_networking
   $modules/network/netconf/netconf_get.py: wisotzky $team_networking
+  $modules/network/netconf/netconf_rpc.py: wisotzky $team_networking
   $modules/network/netscaler/: $team_netscaler
   $modules/network/netvisor/: $team_netvisor
   $modules/network/nuage/: pdellaert

lib/ansible/module_utils/network/netconf/netconf.py

@@ -48,13 +48,13 @@ def get_capabilities(module):
     return module._netconf_capabilities
 
 
-def lock_configuration(x, target=None):
+def lock_configuration(module, target=None):
-    conn = get_connection(x)
+    conn = get_connection(module)
     return conn.lock(target=target)
 
 
-def unlock_configuration(x, target=None):
+def unlock_configuration(module, target=None):
-    conn = get_connection(x)
+    conn = get_connection(module)
     return conn.unlock(target=target)
 
 
@@ -104,3 +104,13 @@ def get(module, filter, lock=False):
             conn.unlock(target='running')
 
     return response
+
+
+def dispatch(module, request):
+    conn = get_connection(module)
+    try:
+        response = conn.dispatch(request)
+    except ConnectionError as e:
+        module.fail_json(msg=to_text(e, errors='surrogate_then_replace').strip())
+
+    return response

lib/ansible/modules/network/netconf/netconf_rpc.py

@@ -0,0 +1,262 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2018, Ansible by Red Hat, inc
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+
+DOCUMENTATION = """
+---
+module: netconf_rpc
+version_added: "2.6"
+author:
+    - "Ganesh Nalawade (@ganeshrn)"
+    - "Sven Wisotzky (@wisotzky)"
+short_description: Execute operations on NETCONF enabled network devices.
+description:
+    - NETCONF is a network management protocol developed and standardized by
+      the IETF. It is documented in RFC 6241.
+    - This module allows the user to execute NETCONF RPC requests as defined
+      by IETF RFC standards as well as proprietary requests.
+options:
+  rpc:
+    description:
+      - This argument specifies the request (name of the operation) to be executed on
+        the remote NETCONF enabled device.
+  xmlns:
+    description:
+      - NETCONF operations not defined in rfc6241 typically require the appropriate
+        XML namespace to be set. In the case the I(request) option is not already
+        provided in XML format, the namespace can be defined by the I(xmlns)
+        option.
+  content:
+    description:
+      - This argument specifies the optional request content (all RPC attributes).
+        The I(content) value can either be provided as XML formatted string or as
+        dictionary.
+  display:
+    description:
+      - Encoding scheme to use when serializing output from the device. The option I(json) will
+        serialize the output as JSON data. If the option value is I(json) it requires jxmlease
+        to be installed on control node. The option I(pretty) is similar to received XML response
+        but is using human readable format (spaces, new lines). The option value I(xml) is similar
+        to received XML response but removes all XML namespaces.
+    choices: ['json', 'pretty', 'xml']
+requirements:
+  - ncclient (>=v0.5.2)
+  - jxmlease
+
+notes:
+  - This module requires the NETCONF system service be enabled on the remote device
+    being managed.
+  - This module supports the use of connection=netconf
+  - To execute C(get-config), C(get) or C(edit-config) requests it is recommended
+    to use the Ansible I(netconf_get) and I(netconf_config) modules.
+"""
+
+EXAMPLES = """
+- name: lock candidate
+  netconf_rpc:
+    rpc: lock
+    content:
+      target:
+        candidate:
+
+- name: unlock candidate
+  netconf_rpc:
+    rpc: unlock
+    xmlns: "urn:ietf:params:xml:ns:netconf:base:1.0"
+    content: "{'target': {'candidate': None}}"
+
+- name: discard changes
+  netconf_rpc:
+    rpc: discard-changes
+
+- name: get-schema
+  netconf_rpc:
+    rpc: get-schema
+    xmlns: urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring
+    content:
+      identifier: ietf-netconf
+      version: "2011-06-01"
+
+- name: copy running to startup
+  netconf_rpc:
+    rpc: copy-config
+    content:
+      source:
+        running:
+      target:
+        startup:
+
+- name: get schema list with JSON output
+  netconf_rpc:
+    rpc: get
+    content: |
+      <filter>
+        <netconf-state xmlns="urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring">
+          <schemas/>
+        </netconf-state>
+      </filter>
+    display: json
+
+- name: get schema using XML request
+  netconf_rpc:
+    rpc: "get-schema"
+    xmlns: "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring"
+    content: |
+      <identifier>ietf-netconf-monitoring</identifier>
+      <version>2010-10-04</version>
+    display: json
+"""
+
+RETURN = """
+stdout:
+  description: The raw XML string containing configuration or state data
+               received from the underlying ncclient library.
+  returned: always apart from low-level errors (such as action plugin)
+  type: string
+  sample: '...'
+stdout_lines:
+  description: The value of stdout split into a list
+  returned: always apart from low-level errors (such as action plugin)
+  type: list
+  sample: ['...', '...']
+output:
+  description: Based on the value of display option will return either the set of
+               transformed XML to JSON format from the RPC response with type dict
+               or pretty XML string response (human-readable) or response with
+               namespace removed from XML string.
+  returned: when the display format is selected as JSON it is returned as dict type, if the
+            display format is xml or pretty pretty it is retured as a string apart from low-level
+            errors (such as action plugin).
+  type: complex
+  contains:
+    formatted_output:
+      - Contains formatted response received from remote host as per the value in display format.
+"""
+
+import ast
+
+try:
+    from lxml.etree import tostring
+except ImportError:
+    from xml.etree.ElementTree import tostring
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.network.netconf.netconf import dispatch
+from ansible.module_utils.network.common.netconf import remove_namespaces
+
+try:
+    import jxmlease
+    HAS_JXMLEASE = True
+except ImportError:
+    HAS_JXMLEASE = False
+
+
+def get_xml_request(module, request, xmlns, content):
+    if content is None:
+        if xmlns is None:
+            return '<%s/>' % request
+        else:
+            return '<%s xmlns="%s"/>' % (request, xmlns)
+
+    if isinstance(content, str):
+        content = content.strip()
+
+        if content.startswith('<') and content.endswith('>'):
+            # assumption content contains already XML payload
+            if xmlns is None:
+                return '<%s>%s</%s>' % (request, content, request)
+            else:
+                return '<%s xmlns="%s">%s</%s>' % (request, xmlns, content, request)
+
+        try:
+            # trying if content contains dict
+            content = ast.literal_eval(content)
+        except:
+            module.fail_json(msg='unsupported content value `%s`' % content)
+
+    if isinstance(content, dict):
+        if not HAS_JXMLEASE:
+            module.fail_json(msg='jxmlease is required to convert RPC content to XML '
+                                 'but does not appear to be installed. '
+                                 'It can be installed using `pip install jxmlease`')
+
+        payload = jxmlease.XMLDictNode(content).emit_xml(pretty=False, full_document=False)
+        if xmlns is None:
+            return '<%s>%s</%s>' % (request, payload, request)
+        else:
+            return '<%s xmlns="%s">%s</%s>' % (request, xmlns, payload, request)
+
+    module.fail_json(msg='unsupported content data-type `%s`' % type(content).__name__)
+
+
+def main():
+    """entry point for module execution
+    """
+    argument_spec = dict(
+        rpc=dict(type="str", required=True),
+        xmlns=dict(type="str"),
+        content=dict(),
+        display=dict(choices=['json', 'pretty', 'xml'])
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec,
+                           supports_check_mode=True)
+
+    rpc = module.params['rpc']
+    xmlns = module.params['xmlns']
+    content = module.params['content']
+    display = module.params['display']
+
+    if rpc is None:
+        module.fail_json(msg='argument `rpc` must not be None')
+
+    rpc = rpc.strip()
+    if len(rpc) == 0:
+        module.fail_json(msg='argument `rpc` must not be empty')
+
+    if rpc in ['close-session']:
+        # explicit close-session is not allowed, as this would make the next
+        # NETCONF operation to the same host fail
+        module.fail_json(msg='unsupported operation `%s`' % rpc)
+
+    if display == 'json' and not HAS_JXMLEASE:
+        module.fail_json(msg='jxmlease is required to display response in json format'
+                             'but does not appear to be installed. '
+                             'It can be installed using `pip install jxmlease`')
+
+    xml_req = get_xml_request(module, rpc, xmlns, content)
+    response = dispatch(module, xml_req)
+
+    xml_resp = tostring(response)
+    output = None
+
+    if display == 'xml':
+        output = remove_namespaces(xml_resp)
+    elif display == 'json':
+        try:
+            output = jxmlease.parse(xml_resp)
+        except:
+            raise ValueError(xml_resp)
+    elif display == 'pretty':
+        output = tostring(response, pretty_print=True)
+
+    result = {
+        'stdout': xml_resp,
+        'output': output
+    }
+
+    module.exit_json(**result)
+
+if __name__ == '__main__':
+    main()

lib/ansible/plugins/netconf/__init__.py

@@ -32,6 +32,11 @@ try:
 except ImportError:
     raise AnsibleError("ncclient is not installed")
 
+try:
+    from lxml.etree import Element, SubElement, tostring, fromstring
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement, tostring, fromstring
+
 
 def ensure_connected(func):
     @wraps(func)
@@ -106,7 +111,7 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
             resp = self.m.rpc(obj)
             return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
         except RPCError as exc:
-            msg = exc.data_xml if hasattr(exc, 'data_xml') else exc.xml
+            msg = exc.xml
             raise Exception(to_xml(msg))
 
     @ensure_connected
@@ -174,6 +179,15 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         resp = self.m.copy_config(*args, **kwargs)
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
+    @ensure_connected
+    def dispatch(self, request):
+        """Execute operation on the remote device
+        :request: is the rpc request including attributes as XML string
+        """
+        req = fromstring(request)
+        resp = self.m.dispatch(req)
+        return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
+
     @ensure_connected
     def lock(self, target=None):
         """
@@ -228,13 +242,6 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         resp = self.m.commit(*args, **kwargs)
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
-    @ensure_connected
-    def validate(self, *args, **kwargs):
-        """Validate the contents of the specified configuration.
-           :source: name of configuration data store"""
-        resp = self.m.validate(*args, **kwargs)
-        return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
-
     @ensure_connected
     def get_schema(self, *args, **kwargs):
         """Retrieves the required schema from the device
@@ -277,15 +284,15 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
     def get_device_operations(self, server_capabilities):
         operations = {}
         capabilities = '\n'.join(server_capabilities)
-        operations['supports_commit'] = True if ':candidate' in capabilities else False
+        operations['supports_commit'] = ':candidate' in capabilities
-        operations['supports_defaults'] = True if ':with-defaults' in capabilities else False
+        operations['supports_defaults'] = ':with-defaults' in capabilities
-        operations['supports_confirm_commit'] = True if ':confirmed-commit' in capabilities else False
+        operations['supports_confirm_commit'] = ':confirmed-commit' in capabilities
-        operations['supports_startup'] = True if ':startup' in capabilities else False
+        operations['supports_startup'] = ':startup' in capabilities
-        operations['supports_xpath'] = True if ':xpath' in capabilities else False
+        operations['supports_xpath'] = ':xpath' in capabilities
-        operations['supports_writeable_running'] = True if ':writable-running' in capabilities else False
+        operations['supports_writable_running'] = ':writable-running' in capabilities
 
         operations['lock_datastore'] = []
-        if operations['supports_writeable_running']:
+        if operations['supports_writable_running']:
             operations['lock_datastore'].append('running')
 
         if operations['supports_commit']:

test/integration/targets/netconf_rpc/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+testcase: "*"

test/integration/targets/netconf_rpc/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - { role: prepare_junos_tests, when: ansible_network_os == 'junos' }
+  - { role: prepare_iosxr_tests, when: ansible_network_os == 'iosxr' }

test/integration/targets/netconf_rpc/tasks/iosxr.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/iosxr"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }} ansible_connection=netconf"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_rpc/tasks/junos.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/junos"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }} ansible_connection=netconf"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_rpc/tasks/main.yaml

@@ -0,0 +1,4 @@
+---
+- { include: junos.yaml, when: ansible_network_os == 'junos', tags: ['netconf'] }
+- { include: iosxr.yaml, when: ansible_network_os == 'iosxr', tags: ['netconf'] }
+- { include: sros.yaml, when: ansible_network_os == 'sros', tags: ['netconf'] }

test/integration/targets/netconf_rpc/tasks/sros.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/sros"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }} ansible_connection=netconf"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_rpc/tests/iosxr/basic.yaml

@@ -0,0 +1,8 @@
+---
+- debug: msg="START netconf_rpc iosxr/basic.yaml on connection={{ ansible_connection }}"
+
+- name: discard changes
+  netconf_rpc:
+    rpc: discard-changes
+
+- debug: msg="END netconf_rpc iosxr/basic.yaml on connection={{ ansible_connection }}"

test/integration/targets/netconf_rpc/tests/junos/basic.yaml

@@ -0,0 +1,8 @@
+---
+- debug: msg="START netconf_rpc junos/basic.yaml on connection={{ ansible_connection }}"
+
+- name: discard changes
+  netconf_rpc:
+    rpc: discard-changes
+
+- debug: msg="END netconf_rpc junos/basic.yaml on connection={{ ansible_connection }}"

test/integration/targets/netconf_rpc/tests/sros/basic.yaml

@@ -0,0 +1,188 @@
+---
+- debug: msg="START netconf_rpc sros/basic.yaml on connection={{ ansible_connection }}"
+
+- name: lock candidate (content is dict)
+  netconf_rpc:
+    rpc: lock
+    content:
+      target:
+        candidate:
+  register: result
+  connection: netconf
+
+- name: discard changes (w/o content)
+  netconf_rpc:
+    rpc: discard-changes
+    display: xml
+  register: result
+  connection: netconf
+
+- name: unlock candidate (content is dict as json)
+  netconf_rpc:
+    rpc: unlock
+    xmlns: "urn:ietf:params:xml:ns:netconf:base:1.0"
+    content: "{'target': {'candidate': None}}"
+    display: json
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output']['rpc-reply'] is defined}}"
+      - "{{ result['output']['rpc-reply']['ok'] is defined}}"
+
+- name: validate candidate (content is single line of XML)
+  netconf_rpc:
+    rpc: validate
+    content: "<source><candidate/></source>"
+    display: json
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output']['rpc-reply'] is defined}}"
+      - "{{ result['output']['rpc-reply']['ok'] is defined}}"
+
+- name: copy running to startup
+  netconf_rpc:
+    rpc: copy-config
+    content:
+      source:
+        running:
+      target:
+        startup:
+  register: result
+  connection: netconf
+
+- name: get schema list (content is multiple lines of XML)
+  netconf_rpc:
+    rpc: get
+    content: |
+      <filter>
+        <netconf-state xmlns="urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring">
+          <schemas/>
+        </netconf-state>
+      </filter>
+    display: json
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output']['data'] is defined}}"
+      - "{{ result['output']['data']['netconf-state'] is defined}}"
+      - "{{ result['output']['data']['netconf-state']['schemas'] is defined}}"
+      - "{{ result['output']['data']['netconf-state']['schemas']['schema'] is defined}}"
+
+# The following two test-cases have been validated against a pre-release implementation. 
+# To make this playbook work with the regular Nokia SROS 16.0 release, those test-cases
+# have been commented out. As soon the <get-schema> operation is supported by SROS
+# those test-cases shall be included.
+
+#- name: get-schema
+#  netconf_rpc:
+#    rpc: get-schema
+#    xmlns: urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring
+#    content:
+#      identifier: ietf-netconf
+#      version: "2011-06-01"
+#  register: result
+#  connection: netconf
+
+#- name: get schema using XML request
+#  netconf_rpc:
+#    rpc: "get-schema"
+#    xmlns: "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring"
+#    content: |
+#      <identifier>ietf-netconf-monitoring</identifier>
+#      <version>2010-10-04</version>
+#    display: pretty
+#  register: result
+#  connection: netconf
+
+- name: Failure scenario, unsupported content (xpath value)
+  netconf_rpc:
+    rpc: get
+    content: schemas/schema[identifier=ietf-netconf-monitoring]
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'unsupported content value' in result.msg"
+
+- name: Failure scenario, unsupported content type (list)
+  netconf_rpc:
+    rpc: get
+    content:
+      - value1
+      - value2
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'unsupported content data-type' in result.msg"
+
+- name: Failure scenario, RPC is close-session
+  netconf_rpc:
+    rpc: close-session
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'unsupported operation' in result.msg"
+
+- name: Failure scenario, attribute rpc missing
+  netconf_rpc:
+    display: json
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'missing required arguments' in result.msg"
+
+- name: Failure scenario, attribute rpc is None
+  netconf_rpc:
+    rpc:
+    display: json
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'must not be None' in result.msg"
+
+- name: Failure scenario, attribute rpc is zero-length string
+  netconf_rpc:
+    rpc: ""
+    display: json
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'must not be empty' in result.msg"
+
+- name: Failure scenario, attribute rpc only contains white-spaces
+  netconf_rpc:
+    rpc: "   "
+    display: json
+  register: result
+  connection: netconf
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'must not be empty' in result.msg"
+
+- debug: msg="END netconf_rpc sros/basic.yaml on connection={{ ansible_connection }}"