diff --git a/changelogs/fragments/20260817-helm-oci-registry-progress-on-stdout.yml b/changelogs/fragments/20260817-helm-oci-registry-progress-on-stdout.yml new file mode 100644 index 00000000..71ab839d --- /dev/null +++ b/changelogs/fragments/20260817-helm-oci-registry-progress-on-stdout.yml @@ -0,0 +1,3 @@ +bugfixes: + - helm - skip Helm's OCI registry progress messages when parsing the output of ``helm show chart``. As of Helm 4.2.1 these are printed to stdout instead of stderr (https://github.com/helm/helm/pull/32056), which made installing a chart from an OCI registry fail with a YAML scanner error when the chart version contained a ``+`` (https://github.com/ansible-collections/kubernetes.core/pull/1223). + - helm_template - strip Helm's OCI registry progress messages from the returned ``stdout``, which is documented to contain only the rendered templates (https://github.com/ansible-collections/kubernetes.core/pull/1223). diff --git a/plugins/module_utils/helm.py b/plugins/module_utils/helm.py index 7bac93ea..077b53b4 100644 --- a/plugins/module_utils/helm.py +++ b/plugins/module_utils/helm.py @@ -32,6 +32,53 @@ except ImportError: HAS_YAML = False +# Helm's registry client writes these progress messages to stdout as of helm 4.2.1 +# (helm/helm#32056); earlier versions sent them to stderr. Anything that parses the +# stdout of a helm command which may pull from an OCI registry has to skip them. +# Kept verbatim from helm's pkg/registry/client.go so that a reworded message fails +# to match, and therefore fails loudly, rather than silently eating real output. +REGISTRY_PROGRESS_RE = re.compile(r"^(?:Pulled|Pushed|Digest): \S") + +REGISTRY_UNDERSCORE_SUFFIX = " contains an underscore.\n" + +REGISTRY_UNDERSCORE_MESSAGE = """ +OCI artifact references (e.g. tags) do not support the plus sign (+). To support +storing semantic versions, Helm adopts the convention of changing plus (+) to +an underscore (_) in chart version tags when pushing to a registry and back to +a plus (+) when pulling from a registry. +""" + +REGISTRY_UNDERSCORE_MESSAGE_LINES = len(REGISTRY_UNDERSCORE_MESSAGE.splitlines()) + + +def strip_registry_progress(out): + """ + Drop OCI registry progress messages from the front of a helm command's stdout. + + Helm emits 'Pulled:'/'Pushed:' and 'Digest:' lines, followed - when the + reference contains an underscore - by a free-text explanation that is not + valid YAML. Since helm 4.2.1 these land on stdout ahead of the real output, + so 'helm show chart' can no longer be parsed and 'helm template' no longer + returns only rendered manifests. + """ + lines = out.splitlines(keepends=True) + index = 0 + while index < len(lines): + if REGISTRY_PROGRESS_RE.match(lines[index]): + index += 1 + elif lines[index].endswith(REGISTRY_UNDERSCORE_SUFFIX): + # helm prints its registryUnderscoreMessage right after this line, led + # by a blank line. Anything else is not a message we know about. + start = index + 1 + end = start + REGISTRY_UNDERSCORE_MESSAGE_LINES + if "".join(lines[start:end]) != REGISTRY_UNDERSCORE_MESSAGE: + break + index = end + else: + break + return "".join(lines[index:]) + + def parse_helm_plugin_list(output=None): """ Parse `helm plugin list`, return list of plugins diff --git a/plugins/modules/helm.py b/plugins/modules/helm.py index ef163de3..2f5ef4ee 100644 --- a/plugins/modules/helm.py +++ b/plugins/modules/helm.py @@ -523,6 +523,7 @@ from ansible.module_utils.basic import missing_required_lib from ansible_collections.kubernetes.core.plugins.module_utils.helm import ( AnsibleHelmModule, parse_helm_plugin_list, + strip_registry_progress, ) from ansible_collections.kubernetes.core.plugins.module_utils.helm_args_common import ( HELM_AUTH_ARG_SPEC, @@ -613,7 +614,7 @@ def fetch_chart_info( rc, out, err = module.run_helm_command(inspect_command) - return yaml.safe_load(out) + return yaml.safe_load(strip_registry_progress(out)) def deploy( diff --git a/plugins/modules/helm_template.py b/plugins/modules/helm_template.py index 0c2a91ab..aaedb0c5 100644 --- a/plugins/modules/helm_template.py +++ b/plugins/modules/helm_template.py @@ -228,6 +228,7 @@ except ImportError: from ansible.module_utils.basic import missing_required_lib from ansible_collections.kubernetes.core.plugins.module_utils.helm import ( AnsibleHelmModule, + strip_registry_progress, ) from ansible_collections.kubernetes.core.plugins.module_utils.version import ( LooseVersion, @@ -394,6 +395,9 @@ def main(): if not check_mode: rc, out, err = module.run_helm_command(tmpl_cmd) + # 'stdout' is documented to hold the rendered templates, so keep helm's + # OCI registry progress messages out of it. + out = strip_registry_progress(out) else: out = err = "" rc = 0 diff --git a/tests/integration/targets/helm_oci_registry_progress/aliases b/tests/integration/targets/helm_oci_registry_progress/aliases new file mode 100644 index 00000000..7aa5c29e --- /dev/null +++ b/tests/integration/targets/helm_oci_registry_progress/aliases @@ -0,0 +1,3 @@ +time=10 +helm +helm_template diff --git a/tests/integration/targets/helm_oci_registry_progress/defaults/main.yml b/tests/integration/targets/helm_oci_registry_progress/defaults/main.yml new file mode 100644 index 00000000..e6cb6c03 --- /dev/null +++ b/tests/integration/targets/helm_oci_registry_progress/defaults/main.yml @@ -0,0 +1,20 @@ +--- +# Helm >= 4.2.1 is required for this test to be meaningful: that is the release which +# moved the OCI registry progress messages from stderr to stdout (helm/helm#32056). +helm_oci_version: v4.2.4 + +helm_oci_namespace: helm-oci-registry-progress +test_namespace: "{{ helm_oci_namespace }}" + +# Matches the registry stood up by the setup_helm_registry role. +helm_oci_registry: "localhost:6035" +helm_oci_registry_username: testuser +helm_oci_registry_password: 'pass123!' +helm_oci_repo_path: testing + +helm_oci_chart_name: test-chart-plus-version +# '+' is not allowed in an OCI tag, so Helm stores this as '1.0.0_build.1'. +helm_oci_chart_version: 1.0.0+build.1 +# A version without build metadata, to cover the case where the progress messages +# happen to be valid YAML and so corrupt the parsed output silently. +helm_oci_chart_version_plain: 1.0.1 diff --git a/tests/integration/targets/helm_oci_registry_progress/files/test-chart-plus-version/Chart.yaml b/tests/integration/targets/helm_oci_registry_progress/files/test-chart-plus-version/Chart.yaml new file mode 100644 index 00000000..4609910f --- /dev/null +++ b/tests/integration/targets/helm_oci_registry_progress/files/test-chart-plus-version/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: test-chart-plus-version +description: A chart whose version carries semver build metadata +type: application +version: 1.0.0+build.1 +appVersion: "default" diff --git a/tests/integration/targets/helm_oci_registry_progress/files/test-chart-plus-version/templates/configmap.yaml b/tests/integration/targets/helm_oci_registry_progress/files/test-chart-plus-version/templates/configmap.yaml new file mode 100644 index 00000000..decb0e28 --- /dev/null +++ b/tests/integration/targets/helm_oci_registry_progress/files/test-chart-plus-version/templates/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-chart-plus-version-configmap +data: + myValue: test diff --git a/tests/integration/targets/helm_oci_registry_progress/meta/main.yml b/tests/integration/targets/helm_oci_registry_progress/meta/main.yml new file mode 100644 index 00000000..745901b7 --- /dev/null +++ b/tests/integration/targets/helm_oci_registry_progress/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - remove_namespace + - setup_helm_registry diff --git a/tests/integration/targets/helm_oci_registry_progress/tasks/main.yml b/tests/integration/targets/helm_oci_registry_progress/tasks/main.yml new file mode 100644 index 00000000..ab6ac5da --- /dev/null +++ b/tests/integration/targets/helm_oci_registry_progress/tasks/main.yml @@ -0,0 +1,158 @@ +--- +# Regression test for https://github.com/ansible-collections/kubernetes.core/issues/1222 +# +# Helm >= 4.2.1 writes OCI registry progress messages to stdout rather than stderr +# (helm/helm#32056): 'Pulled:', 'Digest:' and, when the reference contains an +# underscore, a free-text explanation. Helm rewrites '+' to '_' in OCI tags, so a +# chart version carrying semver build metadata is what triggers that explanation, +# which is not valid YAML. +# +# The helm module parses the stdout of 'helm show chart', and helm_template returns +# the stdout of 'helm template' as its documented result, so both have to skip those +# messages. Without the fix, the helm task below fails with +# "could not find expected ':'". +- name: Test OCI charts whose version contains build metadata + vars: + chart_ref: "oci://{{ helm_oci_registry }}/{{ helm_oci_repo_path }}/{{ helm_oci_chart_name }}" + block: + - name: "Install Helm {{ helm_oci_version }}" + ansible.builtin.include_role: + name: install_helm + vars: + helm_version: "{{ helm_oci_version }}" + + - name: Create temporary directory to hold the chart + ansible.builtin.tempfile: + state: directory + suffix: .chart + register: _tmpd + + - name: Copy the test chart + ansible.builtin.copy: + src: "{{ helm_oci_chart_name }}" + dest: "{{ _tmpd.path }}/" + + - name: Log into the helm registry + ansible.builtin.command: >- + {{ helm_binary }} registry login --plain-http + -u {{ helm_oci_registry_username }} -p {{ helm_oci_registry_password }} + {{ helm_oci_registry }} + + - name: Package and push both chart versions + ansible.builtin.command: "{{ item }}" + loop: + - >- + {{ helm_binary }} package {{ _tmpd.path }}/{{ helm_oci_chart_name }} + --version {{ helm_oci_chart_version }} --destination {{ _tmpd.path }} + - >- + {{ helm_binary }} package {{ _tmpd.path }}/{{ helm_oci_chart_name }} + --version {{ helm_oci_chart_version_plain }} --destination {{ _tmpd.path }} + - >- + {{ helm_binary }} push --plain-http + {{ _tmpd.path }}/{{ helm_oci_chart_name }}-{{ helm_oci_chart_version }}.tgz + oci://{{ helm_oci_registry }}/{{ helm_oci_repo_path }} + - >- + {{ helm_binary }} push --plain-http + {{ _tmpd.path }}/{{ helm_oci_chart_name }}-{{ helm_oci_chart_version_plain }}.tgz + oci://{{ helm_oci_registry }}/{{ helm_oci_repo_path }} + register: _push + + - name: Ensure Helm stored the build metadata version with an underscore + ansible.builtin.assert: + that: + - "'1.0.0_build.1' in (_push.results[2].stdout + _push.results[2].stderr)" + fail_msg: >- + Expected Helm to rewrite '+' to '_' when pushing the chart tag. Without + that rewrite this target no longer covers the case it was written for. + + # helm - the version with build metadata is the case that failed outright. + - name: Install the chart from the OCI registry + kubernetes.core.helm: + binary_path: "{{ helm_binary }}" + name: "{{ helm_oci_chart_name }}" + chart_ref: "{{ chart_ref }}" + chart_version: "{{ helm_oci_chart_version }}" + release_namespace: "{{ helm_oci_namespace }}" + create_namespace: true + plain_http: true + register: _install + + - name: Ensure the release reports the chart version verbatim + ansible.builtin.assert: + that: + - _install is changed + - _install.status.chart == helm_oci_chart_name ~ '-' ~ helm_oci_chart_version + + - name: Install the same chart again + kubernetes.core.helm: + binary_path: "{{ helm_binary }}" + name: "{{ helm_oci_chart_name }}" + chart_ref: "{{ chart_ref }}" + chart_version: "{{ helm_oci_chart_version }}" + release_namespace: "{{ helm_oci_namespace }}" + plain_http: true + register: _reinstall + + - name: Ensure the module is idempotent + ansible.builtin.assert: + that: + - _reinstall is not changed + + # helm_template - stdout is documented to hold only the rendered templates. Both + # chart versions are checked: the plain one covers the case where the progress + # messages are valid YAML and would otherwise be parsed as a manifest. + - name: Render the chart from the OCI registry + kubernetes.core.helm_template: + binary_path: "{{ helm_binary }}" + chart_ref: "{{ chart_ref }}" + chart_version: "{{ item }}" + release_name: "{{ helm_oci_chart_name }}" + plain_http: true + loop: + - "{{ helm_oci_chart_version }}" + - "{{ helm_oci_chart_version_plain }}" + register: _template + + - name: Ensure stdout holds the rendered manifests and nothing else + ansible.builtin.assert: + that: + - "'Pulled:' not in item.stdout" + - "'Digest:' not in item.stdout" + - "'contains an underscore' not in item.stdout" + - item.stdout.startswith('---') + - (item.stdout | from_yaml_all | list | map(attribute='kind') | list) == ['ConfigMap'] + fail_msg: "helm_template stdout was not only rendered manifests: {{ item.stdout[:200] }}" + loop: "{{ _template.results }}" + loop_control: + label: "{{ item.item }}" + + always: + - name: Uninstall the release + kubernetes.core.helm: + binary_path: "{{ helm_binary }}" + name: "{{ helm_oci_chart_name }}" + release_namespace: "{{ helm_oci_namespace }}" + state: absent + when: helm_binary is defined + ignore_errors: true + + - name: Remove the test namespace + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + name: "{{ helm_oci_namespace }}" + state: absent + wait: true + ignore_errors: true + + - name: Log out of the helm registry + ansible.builtin.command: "{{ helm_binary }} registry logout {{ helm_oci_registry }}" + when: helm_binary is defined + ignore_errors: true + + - name: Remove the temporary directory + ansible.builtin.file: + path: "{{ _tmpd.path }}" + state: absent + when: _tmpd is defined and _tmpd.path is defined + ignore_errors: true diff --git a/tests/unit/module_utils/helm/test_strip_registry_progress.py b/tests/unit/module_utils/helm/test_strip_registry_progress.py new file mode 100644 index 00000000..df0f8b97 --- /dev/null +++ b/tests/unit/module_utils/helm/test_strip_registry_progress.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# Copyright: (c) 2026, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import pytest +import yaml +from ansible_collections.kubernetes.core.plugins.module_utils.helm import ( + strip_registry_progress, +) + +# What helm's registry client prints, verbatim. Since helm 4.2.1 this goes to +# stdout ahead of the command's real output (helm/helm#32056). +PULLED = "Pulled: registry.example.com/charts/demochart:109.0.1_up1.0.2\n" +DIGEST = ( + "Digest: sha256:41e9b37f85fa7f5df5fc7f2b7b068eb56ae1592784eee28f535563b599850249\n" +) +PUSHED = "Pushed: registry.example.com/charts/demochart:109.0.1_up1.0.2\n" +UNDERSCORE_WARNING = ( + "registry.example.com/charts/demochart:109.0.1_up1.0.2 contains an underscore.\n" + "\n" + "OCI artifact references (e.g. tags) do not support the plus sign (+). To support\n" + "storing semantic versions, Helm adopts the convention of changing plus (+) to\n" + "an underscore (_) in chart version tags when pushing to a registry and back to\n" + "a plus (+) when pulling from a registry.\n" +) + +# 'helm show chart' output. Note that helm marshals Chart.yaml with sorted keys, so +# 'annotations' precedes 'apiVersion' and the metadata does not reliably start with +# a fixed key. +CHART_YAML = ( + "annotations:\n" + " category: Demo\n" + "apiVersion: v2\n" + "appVersion: 1.16.0\n" + "description: A Helm chart for Kubernetes\n" + "name: demochart\n" + "type: application\n" + "version: 109.0.1+up1.0.2\n" +) + +# 'helm template' output. +MANIFESTS = ( + "---\n" + "# Source: demochart/templates/serviceaccount.yaml\n" + "apiVersion: v1\n" + "kind: ServiceAccount\n" + "metadata:\n" + " name: rel-demochart\n" +) + +# 'helm template --output-dir' output. +WROTE = ( + "wrote outdir/demochart/templates/serviceaccount.yaml\n" + "wrote outdir/demochart/templates/service.yaml\n" +) + + +@pytest.mark.parametrize( + "payload", + [CHART_YAML, MANIFESTS, WROTE, ""], + ids=["show_chart", "template", "template_output_dir", "empty"], +) +@pytest.mark.parametrize( + "progress", + [ + "", + PULLED + DIGEST, + PULLED + DIGEST + UNDERSCORE_WARNING, + PUSHED + DIGEST + UNDERSCORE_WARNING, + ], + ids=["none", "pulled_digest", "with_warning", "pushed_with_warning"], +) +def test_progress_is_stripped_and_payload_is_untouched(progress, payload): + assert strip_registry_progress(progress + payload) == payload + + +def test_show_chart_output_is_parseable_once_stripped(): + # Unstripped, the free-text warning makes this invalid YAML. + out = PULLED + DIGEST + UNDERSCORE_WARNING + CHART_YAML + with pytest.raises(yaml.YAMLError): + yaml.safe_load(out) + + chart_info = yaml.safe_load(strip_registry_progress(out)) + assert chart_info["name"] == "demochart" + assert chart_info["version"] == "109.0.1+up1.0.2" + assert chart_info["annotations"] == {"category": "Demo"} + + +def test_pulled_and_digest_alone_parse_as_yaml_but_pollute_the_result(): + # Without the warning the output is still valid YAML, so the pollution is silent. + assert yaml.safe_load(PULLED + DIGEST + CHART_YAML)["Pulled"] + assert "Pulled" not in yaml.safe_load( + strip_registry_progress(PULLED + DIGEST + CHART_YAML) + ) + + +def test_reworded_warning_is_left_alone(): + # A message helm no longer emits verbatim must not be partially consumed: better + # to fail loudly than to silently drop a line of real output. + reworded = UNDERSCORE_WARNING.replace("plus sign (+)", "plus character (+)") + out = PULLED + DIGEST + reworded + CHART_YAML + assert strip_registry_progress(out) == reworded + CHART_YAML + + +def test_progress_like_lines_inside_the_payload_are_kept(): + payload = "description: |\n Digest: not a helm message\n" + CHART_YAML + assert strip_registry_progress(PULLED + payload) == payload