Provide a mechanism to hide fields from output (#629)

Provide a mechanism to hide fields from output

SUMMARY
The k8s and k8s_info modules can be a little noisy in verbose mode, and most of that is due to managedFields.
If we can provide a mechanism to hide managedFields, the output is a lot more useful.
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
k8s, k8s_info
ADDITIONAL INFORMATION
Before
ANSIBLE_COLLECTIONS_PATH=../../.. ansible -m k8s_info -a 'kind=ConfigMap name=hide-fields-cm namespace=hide-fields' localhost 
[WARNING]: No inventory was parsed, only implicit localhost is available
localhost | SUCCESS => {
    "api_found": true,
    "changed": false,
    "resources": [
        {
            "apiVersion": "v1",
            "data": {
                "another": "value",
                "hello": "world"
            },
            "kind": "ConfigMap",
            "metadata": {
                "annotations": {
                    "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"data\":{\"another\":\"value\",\"hello\":\"world\"},\"kind\":\"ConfigMap\",\"metadata\":{\"annotations\":{},\"name\":\"hide-fields-cm\",\"namespace\":\"hide-fields\"}}\n"
                },
                "creationTimestamp": "2023-06-13T01:47:47Z",
                "managedFields": [
                    {
                        "apiVersion": "v1",
                        "fieldsType": "FieldsV1",
                        "fieldsV1": {
                            "f:data": {
                                ".": {},
                                "f:another": {},
                                "f:hello": {}
                            },
                            "f:metadata": {
                                "f:annotations": {
                                    ".": {},
                                    "f:kubectl.kubernetes.io/last-applied-configuration": {}
                                }
                            }
                        },
                        "manager": "kubectl-client-side-apply",
                        "operation": "Update",
                        "time": "2023-06-13T01:47:47Z"
                    }
                ],
                "name": "hide-fields-cm",
                "namespace": "hide-fields",
                "resourceVersion": "2557394",
                "uid": "f233da63-6374-4079-9825-3562c0ed123c"
            }
        }
    ]
}

After
ANSIBLE_COLLECTIONS_PATH=../../.. ansible -m k8s_info -a 'kind=ConfigMap name=hide-fields-cm namespace=hide-fields hidden_fields=metadata.managedFields' localhost
[WARNING]: No inventory was parsed, only implicit localhost is available
localhost | SUCCESS => {
    "api_found": true,
    "changed": false,
    "resources": [
        {
            "apiVersion": "v1",
            "data": {
                "another": "value",
                "hello": "world"
            },
            "kind": "ConfigMap",
            "metadata": {
                "annotations": {
                    "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"data\":{\"another\":\"value\",\"hello\":\"world\"},\"kind\":\"ConfigMap\",\"metadata\":{\"annotations\":{},\"name\":\"hide-fields-cm\",\"namespace\":\"hide-fields\"}}\n"
                },
                "creationTimestamp": "2023-06-13T01:47:47Z",
                "name": "hide-fields-cm",
                "namespace": "hide-fields",
                "resourceVersion": "2557394",
                "uid": "f233da63-6374-4079-9825-3562c0ed123c"
            }
        }
    ]
}

Reviewed-by: Mike Graves <mgraves@redhat.com>
Reviewed-by: Will Thames

changelogs/fragments/629-add-hidden-fields-option.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+  - k8s, k8s_info - add a hidden_fields option to allow fields to be hidden in the results of k8s and k8s_info

plugins/module_utils/k8s/runner.py

@@ -17,6 +17,7 @@ from ansible_collections.kubernetes.core.plugins.module_utils.k8s.resource impor
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.service import (
     K8sService,
     diff_objects,
+    hide_fields,
 )
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.exceptions import (
     ResourceTimeout,
@@ -137,6 +138,7 @@ def perform_action(svc, definition: Dict, params: Dict) -> Dict:
     state = params.get("state", None)
     kind = definition.get("kind")
     api_version = definition.get("apiVersion")
+    hidden_fields = params.get("hidden_fields")
 
     result = {"changed": False, "result": {}}
     instance = {}
@@ -212,7 +214,7 @@ def perform_action(svc, definition: Dict, params: Dict) -> Dict:
             existing = existing.to_dict()
         else:
             existing = {}
-        match, diffs = diff_objects(existing, instance)
+        match, diffs = diff_objects(existing, instance, hidden_fields)
         if match and diffs:
             result.setdefault("warnings", []).append(
                 "No meaningful diff was generated, but the API may not be idempotent "
@@ -222,7 +224,7 @@ def perform_action(svc, definition: Dict, params: Dict) -> Dict:
         if svc.module._diff:
             result["diff"] = diffs
 
-    result["result"] = instance
+    result["result"] = hide_fields(instance, hidden_fields)
     if not success:
         raise ResourceTimeout(
             '"{0}" "{1}": Timed out waiting on resource'.format(

plugins/module_utils/k8s/service.py

@@ -1,6 +1,7 @@
 # Copyright: (c) 2021, Red Hat | Ansible
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
+import copy
 from typing import Any, Dict, List, Optional, Tuple
 
 from ansible_collections.kubernetes.core.plugins.module_utils.hashes import (
@@ -248,6 +249,7 @@ class K8sService:
         wait_timeout: Optional[int] = 120,
         state: Optional[str] = "present",
         condition: Optional[Dict] = None,
+        hidden_fields: Optional[List] = None,
     ) -> Dict:
         resource = self.find_resource(kind, api_version)
         api_found = bool(resource)
@@ -310,7 +312,9 @@ class K8sService:
         instances = resources.get("items") or [resources]
 
         if not wait:
-            result["resources"] = instances
+            result["resources"] = [
+                hide_fields(instance, hidden_fields) for instance in instances
+            ]
             return result
 
         # Now wait for the specified state of any resource instances we have found.
@@ -329,7 +333,7 @@ class K8sService:
                     "Failed to gather information about %s(s) even"
                     " after waiting for %s seconds" % (res.get("kind"), duration)
                 )
-            result["resources"].append(res)
+            result["resources"].append(hide_fields(res, hidden_fields))
         return result
 
     def create(self, resource: Resource, definition: Dict) -> Dict:
@@ -495,7 +499,9 @@ class K8sService:
         return k8s_obj
 
 
-def diff_objects(existing: Dict, new: Dict) -> Tuple[bool, Dict]:
+def diff_objects(
+    existing: Dict, new: Dict, hidden_fields: Optional[list] = None
+) -> Tuple[bool, Dict]:
     result = {}
     diff = recursive_diff(existing, new)
     if not diff:
@@ -517,4 +523,29 @@ def diff_objects(existing: Dict, new: Dict) -> Tuple[bool, Dict]:
     if not set(result["before"]["metadata"].keys()).issubset(ignored_keys):
         return False, result
 
+    result["before"] = hide_fields(result["before"], hidden_fields)
+    result["after"] = hide_fields(result["after"], hidden_fields)
+
     return True, result
+
+
+def hide_fields(definition: dict, hidden_fields: Optional[list]) -> dict:
+    if not hidden_fields:
+        return definition
+    result = copy.deepcopy(definition)
+    for hidden_field in hidden_fields:
+        result = hide_field(result, hidden_field)
+    return result
+
+
+# hide_field is not hugely sophisticated and designed to cope
+# with e.g. status or metadata.managedFields rather than e.g.
+# spec.template.spec.containers[0].env[3].value
+def hide_field(definition: dict, hidden_field: str) -> dict:
+    split = hidden_field.split(".", 1)
+    if split[0] in definition:
+        if len(split) == 2:
+            definition[split[0]] = hide_field(definition[split[0]], split[1])
+        else:
+            del definition[split[0]]
+    return definition

plugins/modules/k8s.py

@@ -185,6 +185,14 @@ options:
     version_added: 2.5.0
     aliases:
     - all
+  hidden_fields:
+    description:
+      - Hide fields matching this option in the result
+      - An example might be C(hidden_fields=[metadata.managedFields])
+      - Only field definitions that don't reference list items are supported (so V(spec.containers[0]) would not work)
+    type: list
+    elements: str
+    version_added: 2.5.0
 
 requirements:
   - "python >= 3.6"
@@ -472,6 +480,7 @@ def argspec():
         type="dict", default=None, options=server_apply_spec()
     )
     argument_spec["delete_all"] = dict(type="bool", default=False, aliases=["all"])
+    argument_spec["hidden_fields"] = dict(type="list", elements="str")
 
     return argument_spec
 

plugins/modules/k8s_info.py

@@ -44,6 +44,14 @@ options:
     type: list
     elements: str
     default: []
+  hidden_fields:
+    description:
+      - Hide fields matching any of the field definitions in the result
+      - An example might be C(hidden_fields=[metadata.managedFields])
+      - Only field definitions that don't reference list items are supported (so V(spec.containers[0]) would not work)
+    type: list
+    elements: str
+    version_added: 2.5.0
 
 extends_documentation_fragment:
   - kubernetes.core.k8s_auth_options
@@ -183,6 +191,7 @@ def execute_module(module, svc):
         wait_sleep=module.params["wait_sleep"],
         wait_timeout=module.params["wait_timeout"],
         condition=module.params["wait_condition"],
+        hidden_fields=module.params["hidden_fields"],
     )
     module.exit_json(changed=False, **facts)
 
@@ -198,6 +207,7 @@ def argspec():
             namespace=dict(),
             label_selectors=dict(type="list", elements="str", default=[]),
             field_selectors=dict(type="list", elements="str", default=[]),
+            hidden_fields=dict(type="list", elements="str"),
         )
     )
     return args

tests/integration/targets/k8s_hide_fields/aliases

@@ -0,0 +1,3 @@
+time=59
+k8s
+k8s_info

tests/integration/targets/k8s_hide_fields/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+test_namespace: "hide-fields"
+hide_fields_namespace: "hide-fields"
+hide_fields_base_configmap:
+  apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: hide-fields-cm
+    namespace: hide-fields
+  data:
+    hello: world
+    another: value

tests/integration/targets/k8s_hide_fields/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+- setup_namespace

tests/integration/targets/k8s_hide_fields/playbook.yaml

@@ -0,0 +1,6 @@
+---
+- connection: local
+  gather_facts: false
+  hosts: localhost
+  roles:
+  - k8s_hide_fields

tests/integration/targets/k8s_hide_fields/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -eux
+export ANSIBLE_CALLBACKS_ENABLED=profile_tasks
+export ANSIBLE_ROLES_PATH=../
+ansible-playbook playbook.yaml "$@"

tests/integration/targets/k8s_hide_fields/tasks/main.yml

@@ -0,0 +1,108 @@
+- block:
+    - name: Creation with hidden fields should work
+      k8s:
+        definition: "{{ hide_fields_base_configmap}}"
+        hidden_fields:
+          - metadata.managedFields
+      register: hf1
+
+    - name: Ensure hidden fields are not present
+      assert:
+        that:
+          - "'managedFields' not in hf1.result['metadata']"
+
+    - name: Running without hidden fields should work
+      k8s:
+        definition: "{{ hide_fields_base_configmap}}"
+
+    - name: Running with missing hidden fields should have no effect
+      k8s:
+        definition: "{{ hide_fields_base_configmap}}"
+        hidden_fields:
+          - does.not.exist
+      register: hf2
+
+    - name: Ensure no change with missing hidden fields
+      assert:
+        that:
+          - not hf2.changed
+
+    - name: Hide status and managed fields
+      k8s:
+        definition: "{{ hide_fields_base_configmap}}"
+        hidden_fields:
+          - status
+          - metadata.managedFields
+      register: hf3
+      diff: true
+
+    - name: Ensure hidden fields are not present
+      assert:
+        that:
+          - "'status' not in hf3.result"
+          - "'managedFields' not in hf3.result['metadata']"
+
+    - name: k8s_info works with hidden fields
+      k8s_info:
+        name: "{{ hide_fields_base_configmap.metadata.name }}"
+        namespace: "{{ hide_fields_base_configmap.metadata.namespace }}"
+        kind: ConfigMap
+        hidden_fields:
+          - metadata.managedFields
+      register: hf4
+
+    - name: Ensure hidden fields are not present
+      assert:
+        that:
+          - hf4.resources | length == 1
+          - "'managedFields' not in hf4.resources[0]['metadata']"
+
+
+    - name: Hiding a changed field should still result in a change
+      k8s:
+        definition: "{{ hide_fields_base_configmap | combine({'data':{'hello':'different'}}) }}"
+        hidden_fields:
+          - data
+          - metadata.managedFields
+      register: hf5
+      diff: true
+
+    - name: Ensure that hidden changed field changed
+      assert:
+        that:
+          - hf5.changed
+
+    - name: Apply works with hidden fields
+      k8s:
+        definition: "{{ hide_fields_base_configmap | combine({'data':{'anew':'value'}}) }}"
+        hidden_fields:
+          - data
+        apply: true
+      register: hf6
+      diff: true
+
+    - name: Ensure that hidden changed field changed
+      assert:
+        that:
+          - hf6.changed
+
+    - name: Hidden field should not show up in deletion
+      k8s:
+        definition: "{{ hide_fields_base_configmap}}"
+        hidden_fields:
+          - status
+        state: absent
+      register: hf7
+
+    - name: Ensure hidden fields are not present
+      assert:
+        that:
+          - "'status' not in hf7.result"
+
+  always:
+    - name: Remove namespace
+      k8s:
+        kind: Namespace
+        name: "{{ hide_fields_namespace }}"
+        state: absent
+      ignore_errors: true