win_exec: refactor PS exec runner (#45334)

* win_exec: refactor PS exec runner

* more changes for PSCore compatibility

* made some changes based on the recent review

* split up module exec scripts for smaller payload

* removed C# module support to focus on just error msg improvement

* cleaned up c# test classifier code

test/runner/lib/classification.py

@@ -25,6 +25,10 @@ from lib.import_analysis import (
     get_python_module_utils_imports,
 )
 
+from lib.csharp_import_analysis import (
+    get_csharp_module_utils_imports,
+)
+
 from lib.powershell_import_analysis import (
     get_powershell_module_utils_imports,
 )
@@ -168,6 +172,7 @@ class PathMapper(object):
         self.units_targets = list(walk_units_targets())
         self.sanity_targets = list(walk_sanity_targets())
         self.powershell_targets = [t for t in self.sanity_targets if os.path.splitext(t.path)[1] == '.ps1']
+        self.csharp_targets = [t for t in self.sanity_targets if os.path.splitext(t.path)[1] == '.cs']
 
         self.units_modules = set(t.module for t in self.units_targets if t.module)
         self.units_paths = set(a for t in self.units_targets for a in t.aliases)
@@ -189,6 +194,7 @@ class PathMapper(object):
 
         self.python_module_utils_imports = {}  # populated on first use to reduce overhead when not needed
         self.powershell_module_utils_imports = {}  # populated on first use to reduce overhead when not needed
+        self.csharp_module_utils_imports = {}  # populated on first use to reduce overhead when not needed
 
     def get_dependent_paths(self, path):
         """
@@ -204,6 +210,9 @@ class PathMapper(object):
             if ext == '.psm1':
                 return self.get_powershell_module_utils_usage(path)
 
+            if ext == '.cs':
+                return self.get_csharp_module_utils_usage(path)
+
         if path.startswith('test/integration/targets/'):
             return self.get_integration_target_usage(path)
 
@@ -247,6 +256,22 @@ class PathMapper(object):
 
         return sorted(self.powershell_module_utils_imports[name])
 
+    def get_csharp_module_utils_usage(self, path):
+        """
+        :type path: str
+        :rtype: list[str]
+        """
+        if not self.csharp_module_utils_imports:
+            display.info('Analyzing C# module_utils imports...')
+            before = time.time()
+            self.csharp_module_utils_imports = get_csharp_module_utils_imports(self.powershell_targets, self.csharp_targets)
+            after = time.time()
+            display.info('Processed %d C# module_utils in %d second(s).' % (len(self.csharp_module_utils_imports), after - before))
+
+        name = os.path.splitext(os.path.basename(path))[0]
+
+        return sorted(self.csharp_module_utils_imports[name])
+
     def get_integration_target_usage(self, path):
         """
         :type path: str
@@ -320,7 +345,7 @@ class PathMapper(object):
                 return {
                     'units': module_name if module_name in self.units_modules else None,
                     'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None,
-                    'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None,
+                    'windows-integration': self.windows_integration_by_module.get(module_name) if ext in ['.cs', '.ps1'] else None,
                     'network-integration': self.network_integration_by_module.get(module_name),
                     FOCUSED_TARGET: True,
                 }
@@ -328,6 +353,9 @@ class PathMapper(object):
             return minimal
 
         if path.startswith('lib/ansible/module_utils/'):
+            if ext == '.cs':
+                return minimal  # already expanded using get_dependent_paths
+
             if ext == '.psm1':
                 return minimal  # already expanded using get_dependent_paths
 

test/runner/lib/csharp_import_analysis.py

@@ -0,0 +1,76 @@
+"""Analyze C# import statements."""
+
+from __future__ import absolute_import, print_function
+
+import os
+import re
+
+from lib.util import (
+    display,
+)
+
+
+def get_csharp_module_utils_imports(powershell_targets, csharp_targets):
+    """Return a dictionary of module_utils names mapped to sets of powershell file paths.
+    :type powershell_targets: list[TestTarget] - C# files
+    :type csharp_targets: list[TestTarget] - PS files
+    :rtype: dict[str, set[str]]
+    """
+
+    module_utils = enumerate_module_utils()
+
+    imports_by_target_path = {}
+
+    for target in powershell_targets:
+        imports_by_target_path[target.path] = extract_csharp_module_utils_imports(target.path, module_utils, False)
+
+    for target in csharp_targets:
+        imports_by_target_path[target.path] = extract_csharp_module_utils_imports(target.path, module_utils, True)
+
+    imports = dict([(module_util, set()) for module_util in module_utils])
+
+    for target_path in imports_by_target_path:
+        for module_util in imports_by_target_path[target_path]:
+            imports[module_util].add(target_path)
+
+    for module_util in sorted(imports):
+        if not imports[module_util]:
+            display.warning('No imports found which use the "%s" module_util.' % module_util)
+
+    return imports
+
+
+def enumerate_module_utils():
+    """Return a list of available module_utils imports.
+    :rtype: set[str]
+    """
+    return set(os.path.splitext(p)[0] for p in os.listdir('lib/ansible/module_utils/csharp') if os.path.splitext(p)[1] == '.cs')
+
+
+def extract_csharp_module_utils_imports(path, module_utils, is_pure_csharp):
+    """Return a list of module_utils imports found in the specified source file.
+    :type path: str
+    :type module_utils: set[str]
+    :rtype: set[str]
+    """
+    imports = set()
+    if is_pure_csharp:
+        pattern = re.compile(r'(?i)^using\s(Ansible\..+);$')
+    else:
+        pattern = re.compile(r'(?i)^#\s*ansiblerequires\s+-csharputil\s+(Ansible\..+)')
+
+    with open(path, 'r') as module_file:
+        for line_number, line in enumerate(module_file, 1):
+            match = re.search(pattern, line)
+
+            if not match:
+                continue
+
+            import_name = match.group(1)
+            if import_name not in module_utils:
+                display.warning('%s:%d Invalid module_utils import: %s' % (path, line_number, import_name))
+                continue
+
+            imports.add(import_name)
+
+    return imports

test/runner/lib/util.py

@@ -489,6 +489,7 @@ def is_binary_file(path):
         '.cfg',
         '.conf',
         '.crt',
+        '.cs',
         '.css',
         '.html',
         '.ini',