[PR #11678/d06c83eb backport][stable-12] etcd3: re-enable and fix tests, add unit tests (#11680)

etcd3: re-enable and fix tests, add unit tests (#11678)

* etcd3: re-enable and fix tests, add unit tests

- Add unit tests for community.general.etcd3 module (12 tests covering
  state=present/absent, idempotency, check mode, and error paths)
- Fix integration test setup: update etcd binary to v3.6.9 (from v3.2.14),
  download from GitHub releases, add health-check retry loop after start
- Work around etcd3 Python library incompatibility with protobuf >= 4.x
  by setting PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
- Update to FQCNs throughout integration tests
- Re-enable both etcd3 and lookup_etcd3 integration targets

Fixes https://github.com/ansible-collections/community.general/issues/322



* improve use of multiple context managers

---------


(cherry picked from commit d06c83eb68)

Co-authored-by: Alexei Znamensky <103110+russoz@users.noreply.github.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

tests/integration/targets/etcd3/aliases

@@ -7,4 +7,3 @@ destructive
 skip/osx
 skip/macos
 skip/freebsd
-disabled  # see https://github.com/ansible-collections/community.general/issues/322

tests/integration/targets/etcd3/tasks/main.yml

@@ -5,14 +5,9 @@
 ####################################################################
 
 # test code for the etcd3 module
-# Copyright (c) 2017,  Jean-Philippe Evrard <jean-philippe@evrard.me>
+# Copyright (c) 2017, Jean-Philippe Evrard <jean-philippe@evrard.me>
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# ============================================================
+- name: Run tests
-
-
-- name: run_tests for supported distros
   include_tasks: run_tests.yml
-  when:
-    - ansible_facts.distribution | lower ~ "-" ~ ansible_facts.distribution_major_version | lower != 'centos-6'

tests/integration/targets/etcd3/tasks/run_tests.yml

@@ -1,81 +1,83 @@
 ---
 # test code for the etcd3 module
-# Copyright (c) 2017,  Jean-Philippe Evrard <jean-philippe@evrard.me>
+# Copyright (c) 2017, Jean-Philippe Evrard <jean-philippe@evrard.me>
-# Copyright 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
+# Copyright (c) 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# ============================================================
+# The etcd3 Python library uses protobuf-generated code that is incompatible
+# with protobuf >= 4.x unless the pure-Python implementation is selected.
+- environment:
+    PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
+  block:
+    - name: Check mode, show need change
+      community.general.etcd3:
+        key: "foo"
+        value: "bar"
+        state: "present"
+      register: _etcd3_prst_chktst
+      check_mode: true
 
-# Integration tests
+    - name: Change to new value
-- name: Check mode, show need change
+      community.general.etcd3:
-  etcd3:
+        key: "foo"
-    key: "foo"
+        value: "bar"
-    value: "bar"
+        state: "present"
-    state: "present"
+      register: _etcd3_prst_chgtst
-  register: _etcd3_prst_chktst
-  check_mode: true
 
-- name: Change to new value
+    - name: Idempotency test, show unchanged
-  etcd3:
+      community.general.etcd3:
-    key: "foo"
+        key: "foo"
-    value: "bar"
+        value: "bar"
-    state: "present"
+        state: "present"
-  register: _etcd3_prst_chgtst
+      register: _etcd3_prst_idmptnttst
 
-- name: Idempotency test, show unchanged.
+    - name: Idempotency test in check mode, show unchanged
-  etcd3:
+      community.general.etcd3:
-    key: "foo"
+        key: "foo"
-    value: "bar"
+        value: "bar"
-    state: "present"
+        state: "present"
-  register: _etcd3_prst_idmptnttst
+      register: _etcd3_prst_idmptntchktst
+      check_mode: true
 
-- name: Idempotency test in check mode, show unchanged
+    - name: Check mode, show need removal of key
-  etcd3:
+      community.general.etcd3:
-    key: "foo"
+        key: "foo"
-    value: "bar"
+        value: "baz"
-    state: "present"
+        state: "absent"
-  register: _etcd3_prst_idmptntchktst
+      register: _etcd3_absnt_chktst
-  check_mode: true
+      check_mode: true
 
-- name: Check mode, show need removal of key
+    - name: Remove foo key
-  etcd3:
+      community.general.etcd3:
-    key: "foo"
+        key: "foo"
-    value: "baz"
+        value: "baz"
-    state: "absent"
+        state: "absent"
-  register: _etcd3_absnt_chktst
+      register: _etcd3_absnt_chgtst
-  check_mode: true
 
-- name: Remove foo key
+    - name: Idempotency test in check mode, show unchanged
-  etcd3:
+      community.general.etcd3:
-    key: "foo"
+        key: "foo"
-    value: "baz"
+        value: "baz"
-    state: "absent"
+        state: "absent"
-  register: _etcd3_absnt_chgtst
+      register: _etcd3_absnt_idmptnttst
+      check_mode: true
 
-- name: Idempotency test in check mode, show unchanged
+    - name: Idempotency test, show unchanged
-  etcd3:
+      community.general.etcd3:
-    key: "foo"
+        key: "foo"
-    value: "baz"
+        value: "baz"
-    state: "absent"
+        state: "absent"
-  register: _etcd3_absnt_idmptnttst
+      register: _etcd3_absnt_idmptntchktst
-  check_mode: true
 
-- name: Idempotency test, show unchanged
+    - name: Check that statuses are as expected
-  etcd3:
+      ansible.builtin.assert:
-    key: "foo"
+        that:
-    value: "baz"
+          - _etcd3_prst_chktst is changed
-    state: "absent"
+          - _etcd3_prst_chgtst is changed
-  register: _etcd3_absnt_idmptntchktst
+          - _etcd3_prst_idmptnttst is not changed
-
+          - _etcd3_prst_idmptntchktst is not changed
-- name: Checking the status are expected
+          - _etcd3_absnt_chktst is changed
-  assert:
+          - _etcd3_absnt_chgtst is changed
-    that:
+          - _etcd3_absnt_idmptnttst is not changed
-      - _etcd3_prst_chktst is changed
+          - _etcd3_absnt_idmptntchktst is not changed
-      - _etcd3_prst_chgtst is changed
-      - _etcd3_prst_idmptnttst is not changed
-      - _etcd3_prst_idmptntchktst is not changed
-      - _etcd3_absnt_chktst is changed
-      - _etcd3_absnt_chgtst is changed
-      - _etcd3_absnt_idmptnttst is not changed
-      - _etcd3_absnt_idmptntchktst is not changed

tests/integration/targets/lookup_etcd3/aliases

@@ -4,9 +4,7 @@
 
 azp/posix/1
 destructive
-needs/file/tests/utils/constraints.txt
 needs/target/setup_etcd3
 skip/osx
 skip/macos
 skip/freebsd
-disabled  # see https://github.com/ansible-collections/community.general/issues/322

tests/integration/targets/lookup_etcd3/runme.sh

@@ -4,6 +4,11 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 set -eux
 
+# The etcd3 Python library uses protobuf-generated code that is incompatible
+# with protobuf >= 4.x unless the pure-Python implementation is selected.
+# This must be set in the controller process so that lookup plugins are affected.
+export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
 ANSIBLE_ROLES_PATH=../ \
     ansible-playbook dependencies.yml -v "$@"
 

tests/integration/targets/lookup_etcd3/tasks/main.yml

@@ -5,12 +5,12 @@
 ####################################################################
 
 # lookup_etcd3 integration tests
-# Copyright 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
+# Copyright (c) 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-- name: put key/values with an etcd prefix
+- name: Put key/values with an etcd prefix
-  etcd3:
+  community.general.etcd3:
     key: "{{ etcd3_prefix }}foo{{ item }}"
     value: "bar{{ item }}"
     state: present
@@ -19,10 +19,11 @@
     - 2
     - 3
 
-- name: put a single key/values in etcd
+- name: Put a single key/value in etcd
-  etcd3:
+  community.general.etcd3:
     key: "{{ etcd3_singlekey }}"
     value: "foobar"
     state: present
 
-- import_tasks: tests.yml
+- name: Import lookup tests
+  ansible.builtin.import_tasks: tests.yml

tests/integration/targets/setup_etcd3/defaults/main.yml

@@ -1,16 +1,12 @@
 ---
 # setup etcd3 for integration tests on module/lookup
-# (c) 2017,  Jean-Philippe Evrard <jean-philippe@evrard.me>
+# Copyright (c) 2017, Jean-Philippe Evrard <jean-philippe@evrard.me>
-# 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
+# Copyright (c) 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
-#
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
-# # Copyright (c) 2018, Ansible Project
+
-#
+etcd3_ver: "v3.6.9"
-etcd3_ver: "v3.2.14"
+etcd3_download_url: "https://github.com/etcd-io/etcd/releases/download/{{ etcd3_ver }}/etcd-{{ etcd3_ver }}-linux-amd64.tar.gz"
-etcd3_download_server: "https://storage.googleapis.com/etcd"
-# etcd3_download_server: "https://github.com/coreos/etcd/releases/download"
-etcd3_download_url: "{{ etcd3_download_server }}/{{ etcd3_ver }}/etcd-{{ etcd3_ver }}-linux-amd64.tar.gz"
 etcd3_download_location: /tmp/etcd-download-test
 etcd3_path: "{{ etcd3_download_location }}/etcd-{{ etcd3_ver }}-linux-amd64"
 

tests/integration/targets/setup_etcd3/tasks/main.yml

@@ -5,89 +5,49 @@
 ####################################################################
 
 # setup etcd3 for integration tests on module/lookup
-# Copyright 2017,  Jean-Philippe Evrard <jean-philippe@evrard.me>
+# Copyright (c) 2017, Jean-Philippe Evrard <jean-philippe@evrard.me>
-# Copyright 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
+# Copyright (c) 2020, SCC France, Eric Belhomme <ebelhomme@fr.scc.com>
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# ============================================================
+- name: Install etcd3 Python library
+  ansible.builtin.pip:
+    name: "{{ etcd3_pip_module }}"
+    state: present
 
-# setup etcd3 for supported distros
+- name: Check if etcdctl is already usable
-- block:
+  ansible.builtin.command: "{{ etcd3_path }}/etcdctl --endpoints=localhost:2379 endpoint health"
+  changed_when: false
+  failed_when: false
+  register: etcd3_health_check
 
-    - include_vars: '{{ item }}'
+- name: Set up etcd3 binary
-      with_first_found:
+  when: etcd3_health_check.rc != 0
-        - files:
+  block:
-            - '{{ ansible_facts.distribution }}-{{ ansible_facts.distribution_major_version }}.yml'
+    - name: Ensure clean download directory
-            - '{{ ansible_facts.distribution }}-{{ ansible_facts.distribution_version }}.yml'
+      ansible.builtin.file:
-            - '{{ ansible_facts.os_family }}-{{ ansible_facts.distribution_major_version }}.yml'
+        path: "{{ etcd3_download_location }}"
-            - '{{ ansible_facts.os_family }}.yml'
+        state: absent
-            - 'default.yml'
-          paths: '../vars'
 
-    - name: Upgrade setuptools python2 module
+    - name: Create download directory
-      pip:
+      ansible.builtin.file:
-        name: setuptools<45
+        path: "{{ etcd3_download_location }}"
-        extra_args: --upgrade
+        state: directory
-        state: present
-      when: python_suffix == ''
 
-    - name: Install etcd3 python modules
+    - name: Download etcd3
-      pip:
+      ansible.builtin.unarchive:
-        name: "{{ etcd3_pip_module }}"
+        src: "{{ etcd3_download_url }}"
-        extra_args: --only-binary grpcio
+        dest: "{{ etcd3_download_location }}"
-        state: present
+        remote_src: true
 
-    # Check if re-installing etcd3 is required
+    - name: Start etcd3
-    - name: Check if etcd3ctl exists for reuse.
+      ansible.builtin.shell: "nohup {{ etcd3_path }}/etcd > /tmp/etcd3.log 2>&1 &"
-      shell: "ETCDCTL_API=3 {{ etcd3_path }}/etcdctl --endpoints=localhost:2379 get foo"
+      changed_when: true
-      args:
+
-        executable: /bin/bash
+    - name: Wait for etcd3 to be ready
+      ansible.builtin.command: "{{ etcd3_path }}/etcdctl --endpoints=localhost:2379 endpoint health"
+      register: etcd3_ready
+      until: etcd3_ready.rc == 0
+      retries: 10
+      delay: 3
       changed_when: false
-      failed_when: false
-      register: _testetcd3ctl
-
-    - block:
-      # Installing etcd3
-        - name: If can't reuse, prepare download folder
-          file:
-            path: "{{ etcd3_download_location }}"
-            state: directory
-          register: _etcddownloadexists
-          when:
-            - _testetcd3ctl.rc != 0
-
-        - name: Delete download folder if already exists (to start clean)
-          file:
-            path: "{{ etcd3_download_location }}"
-            state: absent
-          when:
-            - _etcddownloadexists is not changed
-
-        - name: Recreate download folder if purged
-          file:
-            path: "{{ etcd3_download_location }}"
-            state: directory
-          when:
-            - _etcddownloadexists is not changed
-
-        - name: Download etcd3
-          unarchive:
-            src: "{{ etcd3_download_url }}"
-            dest: "{{ etcd3_download_location }}"
-            remote_src: true
-
-        # Running etcd3 and kill afterwards if it wasn't running before.
-        - name: Run etcd3
-          shell: "{{ etcd3_path }}/etcd &"
-          register: _etcd3run
-          changed_when: true
-
-      #    - name: kill etcd3
-      #      command: "pkill etcd"
-
-      when:
-        - _testetcd3ctl.rc != 0
-
-  when:
-    - ansible_facts.distribution | lower ~ "-" ~ ansible_facts.distribution_major_version | lower != 'centos-6'

tests/integration/targets/setup_etcd3/vars/Suse.yml

@@ -1,8 +0,0 @@
----
-# Copyright (c) Ansible Project
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# SuSE's python 3.6.10 comes with six 1.11.0 as distutil
-# we restrict to etcd3 < 0.11 to avoid pip to try to upgrade six
-etcd3_pip_module: 'etcd3<0.11'

tests/integration/targets/setup_etcd3/vars/default.yml

@@ -1,6 +0,0 @@
----
-# Copyright (c) Ansible Project
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# default should don't touch anything

tests/unit/plugins/modules/test_etcd3.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2026, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock
+
+import pytest
+from ansible_collections.community.internal_test_tools.tests.unit.plugins.modules.utils import set_module_args
+
+from ansible_collections.community.general.plugins.modules import etcd3 as etcd3_module
+
+BASE_ARGS = {
+    "key": "foo",
+    "value": "bar",
+    "state": "present",
+    "host": "localhost",
+    "port": 2379,
+}
+
+
+@pytest.fixture
+def fake_etcd3(mocker):
+    """Inject a mock etcd3 library into the module namespace and enable HAS_ETCD."""
+    mock_lib = MagicMock()
+    mocker.patch.object(etcd3_module, "etcd3", mock_lib, create=True)
+    mocker.patch.object(etcd3_module, "HAS_ETCD", True)
+    return mock_lib
+
+
+def make_client(fake_etcd3, existing_value=None):
+    """Configure fake_etcd3.client() to return a mock with get() returning the given value."""
+    mock_client = MagicMock()
+    if existing_value is not None:
+        mock_client.get.return_value = (existing_value.encode(), MagicMock())
+    else:
+        mock_client.get.return_value = (None, None)
+    fake_etcd3.client.return_value = mock_client
+    return mock_client
+
+
+# ---------------------------------------------------------------------------
+# state=present
+# ---------------------------------------------------------------------------
+
+
+def test_present_new_key(capfd, fake_etcd3):
+    """state=present with a new key: should put and report changed."""
+    mock_client = make_client(fake_etcd3, existing_value=None)
+
+    with pytest.raises(SystemExit), set_module_args(BASE_ARGS):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is True
+    assert result["key"] == "foo"
+    mock_client.put.assert_called_once_with("foo", "bar")
+
+
+def test_present_same_value(capfd, fake_etcd3):
+    """state=present with existing key and same value: no change."""
+    mock_client = make_client(fake_etcd3, existing_value="bar")
+
+    with pytest.raises(SystemExit), set_module_args(BASE_ARGS):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is False
+    assert result["old_value"] == "bar"
+    mock_client.put.assert_not_called()
+
+
+def test_present_different_value(capfd, fake_etcd3):
+    """state=present with existing key and different value: should put and report changed."""
+    mock_client = make_client(fake_etcd3, existing_value="old_value")
+
+    with pytest.raises(SystemExit), set_module_args(BASE_ARGS):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is True
+    assert result["old_value"] == "old_value"
+    mock_client.put.assert_called_once_with("foo", "bar")
+
+
+# ---------------------------------------------------------------------------
+# state=absent
+# ---------------------------------------------------------------------------
+
+
+def test_absent_existing_key(capfd, fake_etcd3):
+    """state=absent with existing key: should delete and report changed."""
+    mock_client = make_client(fake_etcd3, existing_value="bar")
+
+    with pytest.raises(SystemExit), set_module_args(dict(BASE_ARGS, state="absent")):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is True
+    mock_client.delete.assert_called_once_with("foo")
+
+
+def test_absent_nonexistent_key(capfd, fake_etcd3):
+    """state=absent with key not present: no change."""
+    mock_client = make_client(fake_etcd3, existing_value=None)
+
+    with pytest.raises(SystemExit), set_module_args(dict(BASE_ARGS, state="absent")):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is False
+    mock_client.delete.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# check mode
+# ---------------------------------------------------------------------------
+
+
+def test_present_check_mode_new_key(capfd, fake_etcd3):
+    """state=present in check mode with new key: reports changed but no actual put."""
+    mock_client = make_client(fake_etcd3, existing_value=None)
+
+    with pytest.raises(SystemExit), set_module_args(dict(BASE_ARGS, _ansible_check_mode=True)):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is True
+    mock_client.put.assert_not_called()
+
+
+def test_present_check_mode_same_value(capfd, fake_etcd3):
+    """state=present in check mode with same value: no change, no put."""
+    mock_client = make_client(fake_etcd3, existing_value="bar")
+
+    with pytest.raises(SystemExit), set_module_args(dict(BASE_ARGS, _ansible_check_mode=True)):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is False
+    mock_client.put.assert_not_called()
+
+
+def test_absent_check_mode_existing_key(capfd, fake_etcd3):
+    """state=absent in check mode with existing key: reports changed but no actual delete."""
+    mock_client = make_client(fake_etcd3, existing_value="bar")
+
+    with pytest.raises(SystemExit), set_module_args(dict(BASE_ARGS, state="absent", _ansible_check_mode=True)):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is True
+    mock_client.delete.assert_not_called()
+
+
+def test_absent_check_mode_nonexistent_key(capfd, fake_etcd3):
+    """state=absent in check mode with missing key: no change, no delete."""
+    mock_client = make_client(fake_etcd3, existing_value=None)
+
+    with pytest.raises(SystemExit), set_module_args(dict(BASE_ARGS, state="absent", _ansible_check_mode=True)):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["changed"] is False
+    mock_client.delete.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# error paths
+# ---------------------------------------------------------------------------
+
+
+def test_connection_failure(capfd, fake_etcd3):
+    """Connection to etcd cluster fails: module should fail."""
+    fake_etcd3.client.side_effect = Exception("connection refused")
+
+    with pytest.raises(SystemExit), set_module_args(BASE_ARGS):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["failed"] is True
+    assert "Cannot connect to etcd cluster" in result["msg"]
+
+
+def test_get_failure(capfd, fake_etcd3):
+    """etcd.get() raises: module should fail."""
+    mock_client = MagicMock()
+    mock_client.get.side_effect = Exception("read timeout")
+    fake_etcd3.client.return_value = mock_client
+
+    with pytest.raises(SystemExit), set_module_args(BASE_ARGS):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["failed"] is True
+    assert "Cannot reach data" in result["msg"]
+
+
+def test_missing_library(capfd, mocker):
+    """etcd3 library not installed: module should fail."""
+    mocker.patch.object(etcd3_module, "HAS_ETCD", False)
+
+    with pytest.raises(SystemExit), set_module_args(BASE_ARGS):
+        etcd3_module.main()
+
+    out, dummy = capfd.readouterr()
+    result = json.loads(out)
+    assert result["failed"] is True