fixes for stripping (#52930)

function changed to do in place replacement, should be less expensive even with copy as it avoids 'sub copies', can compose with module_args_copy to create replacement for old behavior

  attempt to fix #52910

* handle lists and subdicts correctly
* added  missing exception case, which was not noticed since 'cleaning' was not working
* added comments to clarify exceptions

lib/ansible/executor/task_result.py

@@ -13,6 +13,14 @@ _IGNORE = ('failed', 'skipped')
 _PRESERVE = ('attempts', 'changed', 'retries')
 _SUB_PRESERVE = {'_ansible_delegated_vars': ('ansible_host', 'ansible_port', 'ansible_user', 'ansible_connection')}
 
+# stuff callbacks need
+CLEAN_EXCEPTIONS = (
+    '_ansible_verbose_always',  # for debug and other actions, to always expand data (pretty jsonification)
+    '_ansible_item_label',  # to know actual 'item' variable
+    '_ansible_no_log',  # jic we didnt clean up well enough, DON'T LOG
+    '_ansible_verbose_override',  # controls display of ansible_facts, gathering would be very noise with -v otherwise
+)
+
 
 class TaskResult:
     '''
@@ -137,6 +145,6 @@ class TaskResult:
                     del result._result[remove_key]
 
             # remove almost ALL internal keys, keep ones relevant to callback
-            strip_internal_keys(result._result, exceptions=('_ansible_verbose_always', '_ansible_item_label', '_ansible_no_log'))
+            strip_internal_keys(result._result, exceptions=CLEAN_EXCEPTIONS)
 
         return result

lib/ansible/plugins/callback/__init__.py

@@ -35,7 +35,7 @@ from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins import AnsiblePlugin, get_plugin_class
 from ansible.utils.color import stringc
 from ansible.utils.display import Display
-from ansible.vars.clean import strip_internal_keys
+from ansible.vars.clean import strip_internal_keys, module_response_deepcopy
 
 if PY3:
     # OrderedDict is needed for a backwards compat shim on Python3.x only
@@ -104,7 +104,7 @@ class CallbackBase(AnsiblePlugin):
             indent = 4
 
         # All result keys stating with _ansible_ are internal, so remove them from the result before we output anything.
-        abridged_result = strip_internal_keys(result)
+        abridged_result = strip_internal_keys(module_response_deepcopy(result))
 
         # remove invocation unless specifically wanting it
         if not keep_invocation and self._display.verbosity < 3 and 'invocation' in result:

lib/ansible/plugins/callback/yaml.py

@@ -28,7 +28,7 @@ import sys
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.six import string_types
 from ansible.parsing.yaml.dumper import AnsibleDumper
-from ansible.plugins.callback import CallbackBase, strip_internal_keys
+from ansible.plugins.callback import CallbackBase, strip_internal_keys, module_response_deepcopy
 from ansible.plugins.callback.default import CallbackModule as Default
 
 
@@ -85,7 +85,7 @@ class CallbackModule(Default):
             return json.dumps(dict(censored="The output has been hidden due to the fact that 'no_log: true' was specified for this result"))
 
         # All result keys stating with _ansible_ are internal, so remove them from the result before we output anything.
-        abridged_result = strip_internal_keys(result)
+        abridged_result = strip_internal_keys(module_response_deepcopy(result))
 
         # remove invocation unless specifically wanting it
         if not keep_invocation and self._display.verbosity < 3 and 'invocation' in result:

lib/ansible/plugins/strategy/__init__.py

@@ -49,7 +49,7 @@ from ansible.plugins.loader import action_loader, connection_loader, filter_load
 from ansible.template import Templar
 from ansible.utils.display import Display
 from ansible.utils.vars import combine_vars
-from ansible.vars.clean import strip_internal_keys
+from ansible.vars.clean import strip_internal_keys, module_response_deepcopy
 
 display = Display()
 
@@ -436,7 +436,7 @@ class StrategyBase:
             if original_task.register:
                 host_list = self.get_task_hosts(iterator, original_host, original_task)
 
-                clean_copy = strip_internal_keys(task_result._result)
+                clean_copy = strip_internal_keys(module_response_deepcopy(task_result._result))
                 if 'invocation' in clean_copy:
                     del clean_copy['invocation']
 

lib/ansible/vars/clean.py

@@ -9,11 +9,14 @@ import os
 import re
 
 from ansible import constants as C
-from ansible.module_utils._text import to_text
+from ansible.errors import AnsibleError
 from ansible.module_utils import six
+from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 from ansible.plugins.loader import connection_loader
 from ansible.utils.display import Display
 
+
 display = Display()
 
 
@@ -65,21 +68,32 @@ def module_response_deepcopy(v):
 
 
 def strip_internal_keys(dirty, exceptions=None):
-    '''
-    All keys starting with _ansible_ are internal, so create a copy of the 'dirty' dict
-    and remove them from the clean one before returning it
-    '''
+    # All keys starting with _ansible_ are internal, so change the 'dirty' mapping and remove them.
 
     if exceptions is None:
-        exceptions = ()
-    clean = dirty.copy()
-    for k in dirty.keys():
-        if isinstance(k, six.string_types) and k.startswith('_ansible_'):
-            if k not in exceptions:
-                del clean[k]
-        elif isinstance(dirty[k], dict):
-            clean[k] = strip_internal_keys(dirty[k])
-    return clean
+        exceptions = tuple()
+
+    if isinstance(dirty, MutableSequence):
+
+        for element in dirty:
+            if isinstance(element, (MutableMapping, MutableSequence)):
+                strip_internal_keys(element, exceptions=exceptions)
+
+    elif isinstance(dirty, MutableMapping):
+
+        # listify to avoid updating dict while iterating over it
+        for k in list(dirty.keys()):
+            if isinstance(k, six.string_types):
+                if k.startswith('_ansible_') and k not in exceptions:
+                    del dirty[k]
+                    continue
+
+            if isinstance(dirty[k], (MutableMapping, MutableSequence)):
+                strip_internal_keys(dirty[k], exceptions=exceptions)
+    else:
+        raise AnsibleError("Cannot strip invalid keys from %s" % type(dirty))
+
+    return dirty
 
 
 def remove_internal_keys(data):