Refactor junos modules to Use netconf and cliconf plugins (#32621)

* Fix junos integration test fixes as per connection refactor (#33050)

Refactor netconf connection plugin to work with netconf plugin

* Fix junos integration test fixes as per connection refactor (#33050)

Refactor netconf connection plugin to work with netconf plugin
Fix CI failure
Fix unit test failure
Fix review comments

lib/ansible/module_utils/connection.py

@@ -34,8 +34,7 @@ import traceback
 import uuid
 
 from functools import partial
-
-from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.six import iteritems
 
 
@@ -77,7 +76,7 @@ def request_builder(method, *args, **kwargs):
     reqid = str(uuid.uuid4())
     req = {'jsonrpc': '2.0', 'method': method, 'id': reqid}
 
-    params = list(args) or kwargs or None
+    params = args or kwargs or None
     if params:
         req['params'] = params
 
@@ -92,7 +91,7 @@ class ConnectionError(Exception):
             setattr(self, k, v)
 
 
-class Connection:
+class Connection(object):
 
     def __init__(self, socket_path):
         if socket_path is None:
@@ -107,15 +106,8 @@ class Connection:
                 raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name))
             return partial(self.__rpc__, name)
 
-    def __rpc__(self, name, *args, **kwargs):
-        """Executes the json-rpc and returns the output received
-           from remote device.
-           :name: rpc method to be executed over connection plugin that implements jsonrpc 2.0
-           :args: Ordered list of params passed as arguments to rpc method
-           :kwargs: Dict of valid key, value pairs passed as arguments to rpc method
+    def _exec_jsonrpc(self, name, *args, **kwargs):
 
-           For usage refer the respective connection plugin docs.
-        """
         req = request_builder(name, *args, **kwargs)
         reqid = req['id']
 
@@ -133,6 +125,20 @@ class Connection:
         if response['id'] != reqid:
             raise ConnectionError('invalid json-rpc id received')
 
+        return response
+
+    def __rpc__(self, name, *args, **kwargs):
+        """Executes the json-rpc and returns the output received
+           from remote device.
+           :name: rpc method to be executed over connection plugin that implements jsonrpc 2.0
+           :args: Ordered list of params passed as arguments to rpc method
+           :kwargs: Dict of valid key, value pairs passed as arguments to rpc method
+
+           For usage refer the respective connection plugin docs.
+        """
+
+        response = self._exec_jsonrpc(name, *args, **kwargs)
+
         if 'error' in response:
             err = response.get('error')
             msg = err.get('data') or err['message']

lib/ansible/module_utils/junos.py

@@ -17,13 +17,13 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 import collections
+import json
 from contextlib import contextmanager
 from copy import deepcopy
 
 from ansible.module_utils.basic import env_fallback, return_values
-from ansible.module_utils.netconf import send_request, children
-from ansible.module_utils.netconf import discard_changes, validate
-from ansible.module_utils.six import string_types
+from ansible.module_utils.connection import Connection
+from ansible.module_utils.netconf import NetconfConnection
 from ansible.module_utils._text import to_text
 
 try:
@@ -45,7 +45,7 @@ junos_provider_spec = {
     'password': dict(fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD']), no_log=True),
     'ssh_keyfile': dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
     'timeout': dict(type='int'),
-    'transport': dict()
+    'transport': dict(default='netconf', choices=['cli', 'netconf'])
 }
 junos_argument_spec = {
     'provider': dict(type='dict', options=junos_provider_spec),
@@ -66,8 +66,29 @@ def get_provider_argspec():
     return junos_provider_spec
 
 
-def check_args(module, warnings):
-    pass
+def get_connection(module):
+    if hasattr(module, '_junos_connection'):
+        return module._junos_connection
+
+    capabilities = get_capabilities(module)
+    network_api = capabilities.get('network_api')
+    if network_api == 'cliconf':
+        module._junos_connection = Connection(module._socket_path)
+    elif network_api == 'netconf':
+        module._junos_connection = NetconfConnection(module._socket_path)
+    else:
+        module.fail_json(msg='Invalid connection type %s' % network_api)
+
+    return module._junos_connection
+
+
+def get_capabilities(module):
+    if hasattr(module, '_junos_capabilities'):
+        return module._junos_capabilities
+
+    capabilities = Connection(module._socket_path).get_capabilities()
+    module._junos_capabilities = json.loads(capabilities)
+    return module._junos_capabilities
 
 
 def _validate_rollback_id(module, value):
@@ -96,73 +117,58 @@ def load_configuration(module, candidate=None, action='merge', rollback=None, fo
     if action == 'set' and not format == 'text':
         module.fail_json(msg='format must be text when action is set')
 
+    conn = get_connection(module)
     if rollback is not None:
         _validate_rollback_id(module, rollback)
-        xattrs = {'rollback': str(rollback)}
+        obj = Element('load-configuration', {'rollback': str(rollback)})
+        conn.execute_rpc(tostring(obj))
     else:
-        xattrs = {'action': action, 'format': format}
-
-    obj = Element('load-configuration', xattrs)
-
-    if candidate is not None:
-        lookup = {'xml': 'configuration', 'text': 'configuration-text',
-                  'set': 'configuration-set', 'json': 'configuration-json'}
-
-        if action == 'set':
-            cfg = SubElement(obj, 'configuration-set')
-        else:
-            cfg = SubElement(obj, lookup[format])
-
-        if isinstance(candidate, string_types):
-            if format == 'xml':
-                cfg.append(fromstring(candidate))
-            else:
-                cfg.text = to_text(candidate, encoding='latin-1')
-        else:
-            cfg.append(candidate)
-    return send_request(module, obj)
+        return conn.load_configuration(config=candidate, action=action, format=format)
 
 
-def get_configuration(module, compare=False, format='xml', rollback='0'):
+def get_configuration(module, compare=False, format='xml', rollback='0', filter=None):
     if format not in CONFIG_FORMATS:
         module.fail_json(msg='invalid config format specified')
-    xattrs = {'format': format}
+
+    conn = get_connection(module)
     if compare:
+        xattrs = {'format': format}
         _validate_rollback_id(module, rollback)
         xattrs['compare'] = 'rollback'
         xattrs['rollback'] = str(rollback)
-    return send_request(module, Element('get-configuration', xattrs))
+        reply = conn.execute_rpc(tostring(Element('get-configuration', xattrs)))
+    else:
+        reply = conn.get_configuration(format=format, filter=filter)
+
+    return reply
 
 
-def commit_configuration(module, confirm=False, check=False, comment=None, confirm_timeout=None):
-    obj = Element('commit-configuration')
-    if confirm:
-        SubElement(obj, 'confirmed')
+def commit_configuration(module, confirm=False, check=False, comment=None, confirm_timeout=None, synchronize=False,
+                         at_time=None, exit=False):
+    conn = get_connection(module)
     if check:
-        SubElement(obj, 'check')
-    if comment:
-        subele = SubElement(obj, 'log')
-        subele.text = str(comment)
-    if confirm_timeout:
-        subele = SubElement(obj, 'confirm-timeout')
-        subele.text = str(confirm_timeout)
-    return send_request(module, obj)
+        reply = conn.validate()
+    else:
+        reply = conn.commit(confirmed=confirm, timeout=confirm_timeout, comment=comment, synchronize=synchronize, at_time=at_time)
+
+    return reply
 
 
-def command(module, command, format='text', rpc_only=False):
-    xattrs = {'format': format}
+def command(module, cmd, format='text', rpc_only=False):
+    conn = get_connection(module)
     if rpc_only:
-        command += ' | display xml rpc'
-        xattrs['format'] = 'text'
-    return send_request(module, Element('command', xattrs, text=command))
+        cmd += ' | display xml rpc'
+    return conn.command(command=cmd, format=format)
 
 
 def lock_configuration(x):
-    return send_request(x, Element('lock-configuration'))
+    conn = get_connection(x)
+    return conn.lock()
 
 
 def unlock_configuration(x):
-    return send_request(x, Element('unlock-configuration'))
+    conn = get_connection(x)
+    return conn.unlock()
 
 
 @contextmanager
@@ -174,8 +180,12 @@ def locked_config(module):
         unlock_configuration(module)
 
 
-def get_diff(module, rollback='0'):
+def discard_changes(module, exit=False):
+    conn = get_connection(module)
+    return conn.discard_changes(exit=exit)
 
+
+def get_diff(module, rollback='0'):
     reply = get_configuration(module, compare=True, format='text', rollback=rollback)
     # if warning is received from device diff is empty.
     if isinstance(reply, list):
@@ -187,7 +197,7 @@ def get_diff(module, rollback='0'):
 
 
 def load_config(module, candidate, warnings, action='merge', format='xml'):
-
+    get_connection(module)
     if not candidate:
         return
 
@@ -198,8 +208,7 @@ def load_config(module, candidate, warnings, action='merge', format='xml'):
     if isinstance(reply, list):
         warnings.extend(reply)
 
-    validate(module)
-
+    module._junos_connection.validate()
     return get_diff(module)
 
 

lib/ansible/module_utils/netconf.py

@@ -25,89 +25,63 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-from contextlib import contextmanager
-
-from ansible.module_utils._text import to_bytes, to_text
-from ansible.module_utils.connection import exec_command
+from ansible.module_utils._text import to_text, to_native
+from ansible.module_utils.connection import Connection, ConnectionError
 
 try:
-    from lxml.etree import Element, SubElement, fromstring, tostring
+    from lxml.etree import Element, fromstring
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, fromstring, tostring
+    from xml.etree.ElementTree import Element, fromstring
 
 NS_MAP = {'nc': "urn:ietf:params:xml:ns:netconf:base:1.0"}
 
 
-def send_request(module, obj, check_rc=True, ignore_warning=True):
-    request = to_text(tostring(obj), errors='surrogate_or_strict')
-    rc, out, err = exec_command(module, request)
-    if rc != 0 and check_rc:
-        error_root = fromstring(err)
-        fake_parent = Element('root')
-        fake_parent.append(error_root)
-
-        error_list = fake_parent.findall('.//nc:rpc-error', NS_MAP)
-        if not error_list:
-            module.fail_json(msg=str(err))
-
-        warnings = []
-        for rpc_error in error_list:
-            message = rpc_error.find('./nc:error-message', NS_MAP).text
-            severity = rpc_error.find('./nc:error-severity', NS_MAP).text
-
-            if severity == 'warning' and ignore_warning:
-                warnings.append(message)
-            else:
-                module.fail_json(msg=str(err))
-        return warnings
-    return fromstring(to_bytes(out, errors='surrogate_or_strict'))
+def exec_rpc(module, *args, **kwargs):
+    connection = NetconfConnection(module._socket_path)
+    return connection.execute_rpc(*args, **kwargs)
 
 
-def children(root, iterable):
-    for item in iterable:
-        try:
-            ele = SubElement(ele, item)
-        except NameError:
-            ele = SubElement(root, item)
+class NetconfConnection(Connection):
 
+    def __init__(self, socket_path):
+        super(NetconfConnection, self).__init__(socket_path)
 
-def lock(module, target='candidate'):
-    obj = Element('lock')
-    children(obj, ('target', target))
-    return send_request(module, obj)
+    def __rpc__(self, name, *args, **kwargs):
+        """Executes the json-rpc and returns the output received
+           from remote device.
+           :name: rpc method to be executed over connection plugin that implements jsonrpc 2.0
+           :args: Ordered list of params passed as arguments to rpc method
+           :kwargs: Dict of valid key, value pairs passed as arguments to rpc method
 
+           For usage refer the respective connection plugin docs.
+        """
+        self.check_rc = kwargs.pop('check_rc', True)
+        self.ignore_warning = kwargs.pop('ignore_warning', True)
 
-def unlock(module, target='candidate'):
-    obj = Element('unlock')
-    children(obj, ('target', target))
-    return send_request(module, obj)
+        response = self._exec_jsonrpc(name, *args, **kwargs)
+        if 'error' in response:
+            rpc_error = response['error'].get('data')
+            return self.parse_rpc_error(to_native(rpc_error, errors='surrogate_then_replace'))
 
+        return fromstring(to_native(response['result'], errors='surrogate_then_replace'))
 
-def commit(module):
-    return send_request(module, Element('commit'))
+    def parse_rpc_error(self, rpc_error):
+        if self.check_rc:
+            error_root = fromstring(rpc_error)
+            root = Element('root')
+            root.append(error_root)
 
+            error_list = root.findall('.//nc:rpc-error', NS_MAP)
+            if not error_list:
+                raise ConnectionError(to_text(rpc_error, errors='surrogate_then_replace'))
 
-def discard_changes(module):
-    return send_request(module, Element('discard-changes'))
+            warnings = []
+            for error in error_list:
+                message = error.find('./nc:error-message', NS_MAP).text
+                severity = error.find('./nc:error-severity', NS_MAP).text
 
-
-def validate(module):
-    obj = Element('validate')
-    children(obj, ('source', 'candidate'))
-    return send_request(module, obj)
-
-
-def get_config(module, source='running', filter=None):
-    obj = Element('get-config')
-    children(obj, ('source', source))
-    children(obj, ('filter', filter))
-    return send_request(module, obj)
-
-
-@contextmanager
-def locked_config(module):
-    try:
-        lock(module)
-        yield
-    finally:
-        unlock(module)
+                if severity == 'warning' and self.ignore_warning:
+                    warnings.append(message)
+                else:
+                    raise ConnectionError(to_text(rpc_error, errors='surrogate_then_replace'))
+            return warnings