Refactor junos modules to Use netconf and cliconf plugins (#32621)

* Fix junos integration test fixes as per connection refactor (#33050)

Refactor netconf connection plugin to work with netconf plugin

* Fix junos integration test fixes as per connection refactor (#33050)

Refactor netconf connection plugin to work with netconf plugin
Fix CI failure
Fix unit test failure
Fix review comments

lib/ansible/modules/network/junos/junos_banner.py

@@ -102,7 +102,7 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -141,8 +141,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_command.py

@@ -171,11 +171,11 @@ import re
 import shlex
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args, get_configuration
+from ansible.module_utils.netconf import exec_rpc
+from ansible.module_utils.junos import junos_argument_spec, get_configuration, get_connection, get_capabilities
 from ansible.module_utils.netcli import Conditional, FailedConditionalError
-from ansible.module_utils.netconf import send_request
 from ansible.module_utils.six import string_types, iteritems
-from ansible.module_utils.connection import Connection
+
 
 try:
     from lxml.etree import Element, SubElement, tostring
@@ -203,7 +203,6 @@ def to_lines(stdout):
 def rpc(module, items):
 
     responses = list()
-
     for item in items:
         name = item['name']
         xattrs = item['xattrs']
@@ -241,7 +240,7 @@ def rpc(module, items):
         if fetch_config:
             reply = get_configuration(module, format=xattrs['format'])
         else:
-            reply = send_request(module, element, ignore_warning=False)
+            reply = exec_rpc(module, tostring(element), ignore_warning=False)
 
         if xattrs['format'] == 'text':
             if fetch_config:
@@ -365,16 +364,24 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
+    conn = get_connection(module)
+    capabilities = get_capabilities(module)
 
-    if module.params['provider'] and module.params['provider']['transport'] == 'cli':
+    if capabilities.get('network_api') == 'cliconf':
         if any((module.params['wait_for'], module.params['match'], module.params['rpcs'])):
             module.warn('arguments wait_for, match, rpcs are not supported when using transport=cli')
         commands = module.params['commands']
-        conn = Connection(module)
+
         output = list()
+        display = module.params['display']
         for cmd in commands:
-            output.append(conn.get(cmd))
+            # if display format is not mentioned in command, add the display format
+            # from the modules params
+            if ('display json' not in cmd) and ('display xml' not in cmd):
+                if display and display != 'text':
+                    cmd += ' | display {0}'.format(display)
+            output.append(conn.get(command=cmd))
+
         lines = [out.split('\n') for out in output]
         result = {'changed': False, 'stdout': output, 'stdout_lines': lines}
         module.exit_json(**result)

lib/ansible/modules/network/junos/junos_config.py

@@ -189,11 +189,10 @@ import re
 import json
 
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.netconf import exec_rpc
 from ansible.module_utils.junos import get_diff, load_config, get_configuration
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
-from ansible.module_utils.junos import junos_argument_spec, load_configuration
-from ansible.module_utils.junos import check_args as junos_check_args
-from ansible.module_utils.netconf import send_request
+from ansible.module_utils.junos import junos_argument_spec, load_configuration, get_connection, tostring
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
 
@@ -217,14 +216,12 @@ DEFAULT_COMMENT = 'configured by junos_config'
 
 
 def check_args(module, warnings):
-    junos_check_args(module, warnings)
-
     if module.params['replace'] is not None:
         module.fail_json(msg='argument replace is deprecated, use update')
 
 
-def zeroize(ele):
-    return send_request(ele, Element('request-system-zeroize'))
+def zeroize(module):
+    return exec_rpc(module, tostring(Element('request-system-zeroize')), ignore_warning=False)
 
 
 def rollback(ele, id='0'):

lib/ansible/modules/network/junos/junos_facts.py

@@ -78,10 +78,10 @@ ansible_facts:
   type: dict
 """
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args, get_param
-from ansible.module_utils.junos import get_configuration
+from ansible.module_utils.netconf import exec_rpc
+from ansible.module_utils.junos import junos_argument_spec, get_param
+from ansible.module_utils.junos import get_configuration, get_connection
 from ansible.module_utils.pycompat24 import get_exception
-from ansible.module_utils.netconf import send_request
 from ansible.module_utils.six import iteritems
 
 
@@ -117,7 +117,7 @@ class FactsBase(object):
         return str(output.text).strip()
 
     def rpc(self, rpc):
-        return send_request(self.module, Element(rpc))
+        return exec_rpc(self.module, tostring(Element(rpc)))
 
     def get_text(self, ele, tag):
         try:
@@ -222,7 +222,7 @@ class Interfaces(FactsBase):
     def populate(self):
         ele = Element('get-interface-information')
         SubElement(ele, 'detail')
-        reply = send_request(self.module, ele)
+        reply = exec_rpc(self.module, tostring(ele))
 
         interfaces = {}
 
@@ -309,9 +309,8 @@ def main():
     module = AnsibleModule(argument_spec=argument_spec,
                            supports_check_mode=True)
 
+    get_connection(module)
     warnings = list()
-    check_args(module, warnings)
-
     gather_subset = module.params['gather_subset']
     ofacts = False
 

lib/ansible/modules/network/junos/junos_interface.py

@@ -185,10 +185,10 @@ from copy import deepcopy
 from time import sleep
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.netconf import send_request
+from ansible.module_utils.netconf import exec_rpc
 from ansible.module_utils.network_common import remove_default_spec
 from ansible.module_utils.network_common import conditional
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config, to_param_list
 
@@ -260,8 +260,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:
@@ -338,8 +336,7 @@ def main():
         if result['changed']:
             sleep(item.get('delay'))
 
-        reply = send_request(module, element, ignore_warning=False)
-
+        reply = exec_rpc(module, tostring(element), ignore_warning=False)
         if state in ('up', 'down'):
             admin_status = reply.xpath('interface-information/physical-interface/admin-status')
             if not admin_status or not conditional(state, admin_status[0].text.strip()):
@@ -361,7 +358,7 @@ def main():
                 intf_name = SubElement(element, 'interface-device')
                 intf_name.text = item.get('name')
 
-                reply = send_request(module, element, ignore_warning=False)
+                reply = exec_rpc(module, tostring(element), ignore_warning=False)
                 have_host = [item.text for item in reply.xpath('lldp-neighbors-information/lldp-neighbor-information/lldp-remote-system-name')]
                 have_port = [item.text for item in reply.xpath('lldp-neighbors-information/lldp-neighbor-information/lldp-remote-port-id')]
 

lib/ansible/modules/network/junos/junos_l3_interface.py

@@ -103,7 +103,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config, to_param_list
 
@@ -149,8 +149,6 @@ def main():
                            required_one_of=required_one_of)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_linkagg.py

@@ -161,7 +161,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config, get_configuration
 
@@ -290,8 +290,6 @@ def main():
                            mutually_exclusive=mutually_exclusive)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_lldp.py

@@ -104,7 +104,7 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -156,8 +156,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_lldp_interface.py

@@ -93,7 +93,7 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -120,8 +120,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_logging.py

@@ -139,7 +139,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -214,8 +214,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_netconf.py

@@ -71,8 +71,7 @@ commands:
 import re
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.connection import exec_command
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec, get_connection
 from ansible.module_utils.junos import commit_configuration, discard_changes
 from ansible.module_utils.network_common import to_list
 from ansible.module_utils.six import iteritems
@@ -103,10 +102,10 @@ def parse_port(config):
 
 
 def map_config_to_obj(module):
-    cmd = 'show configuration system services netconf'
-    rc, out, err = exec_command(module, cmd)
-    if rc != 0:
-        module.fail_json(msg='unable to retrieve current config', stderr=err)
+    conn = get_connection(module)
+    out = conn.get(command='show configuration system services netconf')
+    if out is None:
+        module.fail_json(msg='unable to retrieve current config')
     config = str(out).strip()
 
     obj = {'state': 'absent'}
@@ -139,23 +138,16 @@ def map_params_to_obj(module):
 
 
 def load_config(module, config, commit=False):
+    conn = get_connection(module)
 
-    exec_command(module, 'configure')
-
-    for item in to_list(config):
-        rc, out, err = exec_command(module, item)
-        if rc != 0:
-            module.fail_json(msg=str(err))
-
-    exec_command(module, 'top')
-    rc, diff, err = exec_command(module, 'show | compare')
-
+    conn.edit_config(to_list(config) + ['top'])
+    diff = conn.compare_configuration()
     if diff:
         if commit:
-            exec_command(module, 'commit and-quit')
+            commit_configuration(module)
+
         else:
-            for cmd in ['rollback 0', 'exit']:
-                exec_command(module, cmd)
+            discard_changes(module)
 
     return str(diff).strip()
 
@@ -174,8 +166,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False, 'warnings': warnings}
 
     want = map_params_to_obj(module)

lib/ansible/modules/network/junos/junos_rpc.py

@@ -95,8 +95,8 @@ output_lines:
   type: list
 """
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args
-from ansible.module_utils.netconf import send_request
+from ansible.module_utils.netconf import exec_rpc
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.six import iteritems
 
 USE_PERSISTENT_CONNECTION = True
@@ -123,8 +123,6 @@ def main():
                            supports_check_mode=False)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False, 'warnings': warnings}
 
     rpc = str(module.params['rpc']).replace('_', '-')
@@ -154,7 +152,7 @@ def main():
             if value is not True:
                 child.text = value
 
-    reply = send_request(module, element)
+    reply = exec_rpc(module, tostring(element), ignore_warning=False)
 
     result['xml'] = str(tostring(reply))
 

lib/ansible/modules/network/junos/junos_static_route.py

@@ -135,7 +135,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -183,8 +183,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_system.py

@@ -106,7 +106,7 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -151,8 +151,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_user.py

@@ -147,8 +147,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.netconf import send_request
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec, get_connection
 from ansible.module_utils.junos import commit_configuration, discard_changes
 from ansible.module_utils.junos import load_config, locked_config
 from ansible.module_utils.six import iteritems
@@ -167,7 +166,8 @@ def handle_purge(module, want):
     element = Element('system')
     login = SubElement(element, 'login')
 
-    reply = send_request(module, Element('get-configuration'), ignore_warning=False)
+    conn = get_connection(module)
+    reply = conn.execute_rpc(tostring(Element('get-configuration')), ignore_warning=False)
     users = reply.xpath('configuration/system/login/user/name')
     if users:
         for item in users:
@@ -310,8 +310,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False, 'warnings': warnings}
 
     want = map_params_to_obj(module)

lib/ansible/modules/network/junos/junos_vlan.py

@@ -112,7 +112,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -173,8 +173,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings:

lib/ansible/modules/network/junos/junos_vrf.py

@@ -168,7 +168,7 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network_common import remove_default_spec
-from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.junos import commit_configuration, discard_changes, locked_config
 
@@ -216,8 +216,6 @@ def main():
                            mutually_exclusive=mutually_exclusive)
 
     warnings = list()
-    check_args(module, warnings)
-
     result = {'changed': False}
 
     if warnings: