k8s_cp: add support for check_mode, fix doc issue, remove dependency with 'find' when state=from_pod (#512)

k8s_cp: add support for check_mode, fix doc issue, remove dependency with 'find' when state=from_pod

Depends-On: ansible/ansible-zuul-jobs#1635
Depends-On: ansible/ansible-zuul-jobs#1636
Depends-On: #518
Depends-On: #520
SUMMARY

add support for check_mode, closes #380
fix doc issue, closes #485
Remove dependency with 'find' executable when state=from_pod, closes #486

ISSUE TYPE


Bugfix Pull Request
Docs Pull Request
Feature Pull Request

Reviewed-by: Gonéri Le Bouder <goneri@lebouder.net>
Reviewed-by: Mike Graves <mgraves@redhat.com>
Reviewed-by: Bikouo Aubin <None>

changelogs/fragments/512-k8s_cp-add-support-for-check_mode-update-command-for-listing-files-into-pod.yaml

@@ -0,0 +1,5 @@
+---
+bugfixes:
+- k8s_cp - add support for check_mode (https://github.com/ansible-collections/kubernetes.core/issues/380).
+minor_changes:
+- k8s_cp - remove dependency with 'find' executable on remote pod when state=from_pod (https://github.com/ansible-collections/kubernetes.core/issues/486).

plugins/module_utils/copy.py

@@ -64,6 +64,51 @@ class K8SCopy(metaclass=ABCMeta):
         self.container_arg = {}
         if module.params.get("container"):
             self.container_arg["container"] = module.params.get("container")
+        self.check_mode = self.module.check_mode
+
+    def _run_from_pod(self, cmd):
+        try:
+            resp = stream(
+                self.api_instance.connect_get_namespaced_pod_exec,
+                self.name,
+                self.namespace,
+                command=cmd,
+                async_req=False,
+                stderr=True,
+                stdin=False,
+                stdout=True,
+                tty=False,
+                _preload_content=False,
+                **self.container_arg,
+            )
+
+            stderr, stdout = [], []
+            while resp.is_open():
+                resp.update(timeout=1)
+                if resp.peek_stdout():
+                    stdout.extend(resp.read_stdout().rstrip("\n").split("\n"))
+                if resp.peek_stderr():
+                    stderr.extend(resp.read_stderr().rstrip("\n").split("\n"))
+            error = resp.read_channel(ERROR_CHANNEL)
+            resp.close()
+            error = yaml.safe_load(error)
+            return error, stdout, stderr
+        except Exception as e:
+            self.module.fail_json(
+                msg="Error while running/parsing from pod {1}/{2} command='{0}' : {3}".format(
+                    self.namespace, self.name, cmd, to_native(e)
+                )
+            )
+
+    def is_directory_path_from_pod(self, file_path, failed_if_not_exists=True):
+        # check if file exists
+        error, out, err = self._run_from_pod(cmd=["test", "-e", file_path])
+        if error.get("status") != "Success":
+            if failed_if_not_exists:
+                return None, "%s does not exist in remote pod filesystem" % file_path
+            return False, None
+        error, out, err = self._run_from_pod(cmd=["test", "-d", file_path])
+        return error.get("status") == "Success", None
 
     @abstractmethod
     def run(self):
@@ -78,56 +123,74 @@ class K8SCopyFromPod(K8SCopy):
     def __init__(self, module, client):
         super(K8SCopyFromPod, self).__init__(module, client)
         self.is_remote_path_dir = None
-        self.files_to_copy = list()
+        self.files_to_copy = []
+        self._shellname = None
+
+    @property
+    def pod_shell(self):
+        if self._shellname is None:
+            for s in ("/bin/sh", "/bin/bash"):
+                error, out, err = self._run_from_pod(s)
+                if error.get("status") == "Success":
+                    self._shellname = s
+                    break
+        return self._shellname
+
+    def listfiles_with_find(self, path):
+        find_cmd = ["find", path, "-type", "f"]
+        error, files, err = self._run_from_pod(cmd=find_cmd)
+        if error.get("status") != "Success":
+            self.module.fail_json(msg=error.get("message"))
+        return files
+
+    def listfile_with_echo(self, path):
+        echo_cmd = [self.pod_shell, "-c", "echo {path}/* {path}/.*".format(path=path)]
+        error, out, err = self._run_from_pod(cmd=echo_cmd)
+        if error.get("status") != "Success":
+            self.module.fail_json(msg=error.get("message"))
+
+        files = []
+        if out:
+            output = out[0] + " "
+            files = [
+                os.path.join(path, p[:-1])
+                for p in output.split(f"{path}/")
+                if p and p[:-1] not in (".", "..")
+            ]
+
+        result = []
+        for f in files:
+            is_dir, err = self.is_directory_path_from_pod(f)
+            if err:
+                continue
+            if not is_dir:
+                result.append(f)
+                continue
+            result += self.listfile_with_echo(f)
+        return result
 
     def list_remote_files(self):
         """
         This method will check if the remote path is a dir or file
         if it is a directory the file list will be updated accordingly
         """
-        try:
-            find_cmd = ["find", self.remote_path, "-type", "f", "-name", "*"]
-            response = stream(
-                self.api_instance.connect_get_namespaced_pod_exec,
-                self.name,
-                self.namespace,
-                command=find_cmd,
-                stdout=True,
-                stderr=True,
-                stdin=False,
-                tty=False,
-                _preload_content=False,
-                **self.container_arg
-            )
-        except Exception as e:
-            self.module.fail_json(
-                msg="Failed to execute on pod {0}/{1} due to : {2}".format(
-                    self.namespace, self.name, to_native(e)
-                )
-            )
-        stderr = []
-        while response.is_open():
-            response.update(timeout=1)
-            if response.peek_stdout():
-                self.files_to_copy.extend(
-                    response.read_stdout().rstrip("\n").split("\n")
-                )
-            if response.peek_stderr():
-                err = response.read_stderr()
-                if "No such file or directory" in err:
-                    self.module.fail_json(
-                        msg="{0} does not exist in remote pod filesystem".format(
-                            self.remote_path
-                        )
-                    )
-                stderr.append(err)
-        error = response.read_channel(ERROR_CHANNEL)
-        response.close()
-        error = yaml.safe_load(error)
-        if error["status"] != "Success":
-            self.module.fail_json(
-                msg="Failed to execute on Pod due to: {0}".format(error)
+        # check is remote path exists and is a file or directory
+        is_dir, error = self.is_directory_path_from_pod(self.remote_path)
+        if error:
+            self.module.fail_json(msg=error)
+
+        if not is_dir:
+            return [self.remote_path]
+        else:
+            # find executable to list dir with
+            executables = dict(
+                find=self.listfiles_with_find,
+                echo=self.listfile_with_echo,
             )
+            for item in executables:
+                error, out, err = self._run_from_pod(item)
+                if error.get("status") == "Success":
+                    return executables.get(item)(self.remote_path)
 
     def read(self):
         self.stdout = None
@@ -162,40 +225,42 @@ class K8SCopyFromPod(K8SCopy):
         if is_remote_path_dir and os.path.isdir(self.local_path):
             relpath_start = os.path.dirname(self.remote_path)
 
-        for remote_file in self.files_to_copy:
-            dest_file = self.local_path
-            if is_remote_path_dir:
-                dest_file = os.path.join(
-                    self.local_path, os.path.relpath(remote_file, start=relpath_start)
-                )
-                # create directory to copy file in
-                os.makedirs(os.path.dirname(dest_file), exist_ok=True)
+        if not self.check_mode:
+            for remote_file in self.files_to_copy:
+                dest_file = self.local_path
+                if is_remote_path_dir:
+                    dest_file = os.path.join(
+                        self.local_path,
+                        os.path.relpath(remote_file, start=relpath_start),
+                    )
+                    # create directory to copy file in
+                    os.makedirs(os.path.dirname(dest_file), exist_ok=True)
 
-            pod_command = ["cat", remote_file]
-            self.response = stream(
-                self.api_instance.connect_get_namespaced_pod_exec,
-                self.name,
-                self.namespace,
-                command=pod_command,
-                stderr=True,
-                stdin=True,
-                stdout=True,
-                tty=False,
-                _preload_content=False,
-                **self.container_arg
-            )
-            errors = []
-            with open(dest_file, "wb") as fh:
-                while self.response._connected:
-                    self.read()
-                    if self.stdout:
-                        fh.write(self.stdout)
-                    if self.stderr:
-                        errors.append(self.stderr)
-            if errors:
-                self.module.fail_json(
-                    msg="Failed to copy file from Pod: {0}".format("".join(errors))
+                pod_command = ["cat", remote_file]
+                self.response = stream(
+                    self.api_instance.connect_get_namespaced_pod_exec,
+                    self.name,
+                    self.namespace,
+                    command=pod_command,
+                    stderr=True,
+                    stdin=True,
+                    stdout=True,
+                    tty=False,
+                    _preload_content=False,
+                    **self.container_arg,
                 )
+                errors = []
+                with open(dest_file, "wb") as fh:
+                    while self.response._connected:
+                        self.read()
+                        if self.stdout:
+                            fh.write(self.stdout)
+                        if self.stderr:
+                            errors.append(self.stderr)
+                if errors:
+                    self.module.fail_json(
+                        msg="Failed to copy file from Pod: {0}".format("".join(errors))
+                    )
         self.module.exit_json(
             changed=True,
             result="{0} successfully copied locally into {1}".format(
@@ -204,7 +269,7 @@ class K8SCopyFromPod(K8SCopy):
         )
 
     def run(self):
-        self.list_remote_files()
+        self.files_to_copy = self.list_remote_files()
         if self.files_to_copy == []:
             self.module.exit_json(
                 changed=False,
@@ -224,56 +289,6 @@ class K8SCopyToPod(K8SCopy):
         super(K8SCopyToPod, self).__init__(module, client)
         self.files_to_copy = list()
 
-    def run_from_pod(self, command):
-        response = stream(
-            self.api_instance.connect_get_namespaced_pod_exec,
-            self.name,
-            self.namespace,
-            command=command,
-            stderr=True,
-            stdin=False,
-            stdout=True,
-            tty=False,
-            _preload_content=False,
-            **self.container_arg
-        )
-        errors = []
-        while response.is_open():
-            response.update(timeout=1)
-            if response.peek_stderr():
-                errors.append(response.read_stderr())
-        response.close()
-        err = response.read_channel(ERROR_CHANNEL)
-        err = yaml.safe_load(err)
-        response.close()
-        if err["status"] != "Success":
-            self.module.fail_json(
-                msg="Failed to run {0} on Pod.".format(command), errors=errors
-            )
-
-    def is_remote_path_dir(self):
-        pod_command = ["test", "-d", self.remote_path]
-        response = stream(
-            self.api_instance.connect_get_namespaced_pod_exec,
-            self.name,
-            self.namespace,
-            command=pod_command,
-            stdout=True,
-            stderr=True,
-            stdin=False,
-            tty=False,
-            _preload_content=False,
-            **self.container_arg
-        )
-        while response.is_open():
-            response.update(timeout=1)
-        err = response.read_channel(ERROR_CHANNEL)
-        err = yaml.safe_load(err)
-        response.close()
-        if err["status"] == "Success":
-            return True
-        return False
-
     def close_temp_file(self):
         if self.named_temp_file:
             self.named_temp_file.close()
@@ -296,7 +311,12 @@ class K8SCopyToPod(K8SCopy):
             if not os.access(self.local_path, os.R_OK):
                 self.module.fail_json(msg="{0} not readable".format(self.local_path))
 
-        if self.is_remote_path_dir():
+        is_dir, err = self.is_directory_path_from_pod(
+            self.remote_path, failed_if_not_exists=False
+        )
+        if err:
+            self.module.fail_json(msg=err)
+        if is_dir:
             if self.content:
                 self.module.fail_json(
                     msg="When content is specified, remote path should not be an existing directory"
@@ -304,66 +324,67 @@ class K8SCopyToPod(K8SCopy):
             else:
                 dest_file = os.path.join(dest_file, os.path.basename(src_file))
 
-        if self.no_preserve:
-            tar_command = [
-                "tar",
-                "--no-same-permissions",
-                "--no-same-owner",
-                "-xmf",
-                "-",
-            ]
-        else:
-            tar_command = ["tar", "-xmf", "-"]
+        if not self.check_mode:
+            if self.no_preserve:
+                tar_command = [
+                    "tar",
+                    "--no-same-permissions",
+                    "--no-same-owner",
+                    "-xmf",
+                    "-",
+                ]
+            else:
+                tar_command = ["tar", "-xmf", "-"]
 
-        if dest_file.startswith("/"):
-            tar_command.extend(["-C", "/"])
+            if dest_file.startswith("/"):
+                tar_command.extend(["-C", "/"])
 
-        response = stream(
-            self.api_instance.connect_get_namespaced_pod_exec,
-            self.name,
-            self.namespace,
-            command=tar_command,
-            stderr=True,
-            stdin=True,
-            stdout=True,
-            tty=False,
-            _preload_content=False,
-            **self.container_arg
-        )
-        with TemporaryFile() as tar_buffer:
-            with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
-                tar.add(src_file, dest_file)
-            tar_buffer.seek(0)
-            commands = []
-            # push command in chunk mode
-            size = 1024 * 1024
-            while True:
-                data = tar_buffer.read(size)
-                if not data:
-                    break
-                commands.append(data)
+            response = stream(
+                self.api_instance.connect_get_namespaced_pod_exec,
+                self.name,
+                self.namespace,
+                command=tar_command,
+                stderr=True,
+                stdin=True,
+                stdout=True,
+                tty=False,
+                _preload_content=False,
+                **self.container_arg,
+            )
+            with TemporaryFile() as tar_buffer:
+                with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
+                    tar.add(src_file, dest_file)
+                tar_buffer.seek(0)
+                commands = []
+                # push command in chunk mode
+                size = 1024 * 1024
+                while True:
+                    data = tar_buffer.read(size)
+                    if not data:
+                        break
+                    commands.append(data)
 
-            stderr, stdout = [], []
-            while response.is_open():
-                if response.peek_stdout():
-                    stdout.append(response.read_stdout().rstrip("\n"))
-                if response.peek_stderr():
-                    stderr.append(response.read_stderr().rstrip("\n"))
-                if commands:
-                    cmd = commands.pop(0)
-                    response.write_stdin(cmd)
-                else:
-                    break
-            response.close()
-            if stderr:
-                self.close_temp_file()
-                self.module.fail_json(
-                    command=tar_command,
-                    msg="Failed to copy local file/directory into Pod due to: {0}".format(
-                        "".join(stderr)
-                    ),
-                )
-        self.close_temp_file()
+                stderr, stdout = [], []
+                while response.is_open():
+                    if response.peek_stdout():
+                        stdout.append(response.read_stdout().rstrip("\n"))
+                    if response.peek_stderr():
+                        stderr.append(response.read_stderr().rstrip("\n"))
+                    if commands:
+                        cmd = commands.pop(0)
+                        response.write_stdin(cmd)
+                    else:
+                        break
+                response.close()
+                if stderr:
+                    self.close_temp_file()
+                    self.module.fail_json(
+                        command=tar_command,
+                        msg="Failed to copy local file/directory into Pod due to: {0}".format(
+                            "".join(stderr)
+                        ),
+                    )
+            self.close_temp_file()
         if self.content:
             self.module.exit_json(
                 changed=True,

plugins/modules/k8s_cp.py

@@ -119,7 +119,7 @@ EXAMPLES = r"""
     state: from_pod
 
 # copy content into a file in the remote pod
-- name: Copy /tmp/foo from a remote pod to /tmp/bar locally
+- name: Copy content into a file in the remote pod
   kubernetes.core.k8s_cp:
     state: to_pod
     namespace: some-namespace

tests/integration/targets/k8s_copy/defaults/main.yml

@@ -11,3 +11,6 @@ pod_with_two_container:
   container:
   - container-10
   - container-11
+
+pod_without_executable_find:
+  name: openjdk-pod

tests/integration/targets/k8s_copy/tasks/main.yml

@@ -19,6 +19,7 @@
         template: pods_definition.j2
 
     - include_tasks: test_copy_errors.yml
+    - include_tasks: test_check_mode.yml
     - include_tasks: test_copy_file.yml
     - include_tasks: test_multi_container_pod.yml
     - include_tasks: test_copy_directory.yml

tests/integration/targets/k8s_copy/tasks/test_check_mode.yml

@@ -0,0 +1,142 @@
+---
+- name: Create temporary directory for testing
+  tempfile:
+    state: directory
+    suffix: ansible-k8s-copy
+  register: tmpdir
+
+- block:
+  # setup
+  - name: Create local files for testing
+    copy:
+      content: "{{ item.content }}"
+      dest: "{{ local_dir_path }}/{{ item.dest }}"
+    with_items: "{{ test_files }}"
+
+  - name: Create directory into Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: "mkdir {{ pod_dir_path }}"
+
+  - name: Create files into Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: "{{ pod_dir_path }}/{{ item.dest }}"
+      content: "{{ item.content }}"
+      state: to_pod
+    with_items: "{{ test_files }}"
+
+  # Test copy into Pod using check_mode=true
+  - name: Copy text file into remote pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: "{{ pod_dir_path }}/ansible.txt"
+      local_path: "{{ local_dir_path }}/{{ test_files[0].dest }}"
+      state: to_pod
+    check_mode: true
+    register: copy_file
+
+  - name: Ensure task is changed
+    assert:
+      that:
+      - copy_file is changed
+
+  - name: Ensure file does not exists into Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: "test -e {{ pod_dir_path }}/ansible.txt"
+    register: test_file
+    failed_when: test_file.return_code == 0
+
+  - name: Copy directory into Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: "{{ pod_dir_path }}/mydir"
+      local_path: "{{ local_dir_path }}"
+      state: to_pod
+    check_mode: true
+    register: copy_dir
+
+  - name: Ensure task is changed
+    assert:
+      that:
+      - copy_dir is changed
+
+  - name: Ensure file does not exists into Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: "test -e {{ pod_dir_path }}/mydir"
+    register: test_dir
+    failed_when: test_dir.return_code == 0
+
+  # Test copy from pod using check_mode=true
+  - name: Copy file from Pod into local file system
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: "{{ pod_dir_path }}/{{ test_files[0].dest }}"
+      local_path: "{{ local_dir_path }}/ansible.txt"
+      state: from_pod
+    check_mode: true
+    register: copy_file
+
+  - name: Ensure task is changed
+    assert:
+      that:
+      - copy_file is changed
+
+  - name: Ensure file does not exists into local file system
+    stat:
+      path: "{{ local_dir_path }}/ansible.txt"
+    register: testfile
+    failed_when: testfile.stat.exists
+
+  - name: Copy directory from Pod into local file system
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: "{{ pod_dir_path }}"
+      local_path: "{{ local_dir_path }}/mydir"
+      state: from_pod
+    check_mode: true
+    register: _dir
+
+  - name: Ensure task is changed
+    assert:
+      that:
+      - _dir is changed
+
+  - name: Ensure directory does not exist into local file system
+    stat:
+      path: "{{ local_dir_path }}/mydir"
+    register: testdir
+    failed_when: testdir.stat.exists
+
+  vars:
+    local_dir_path: "{{ tmpdir.path }}"
+    pod_dir_path: "/tmp/test_check_mode"
+    test_files:
+    - content: "collection = kubernetes.core"
+      dest: collection.txt
+    - content: "modules = k8s_cp and k8s_exec"
+      dest: modules.txt
+
+  always:
+  - name: Delete temporary directory
+    file:
+      state: absent
+      path: "{{ local_dir_path }}"
+    ignore_errors: true
+
+  - name: Delete temporary directory created into Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: 'rm -r {{ pod_dir_path }}'
+    ignore_errors: true

tests/integration/targets/k8s_copy/tasks/test_copy_directory.yml

@@ -60,6 +60,59 @@
       remote_path: /tmp/data
       local_path: /tmp/data
 
+  # Test copy from Pod where the executable 'find' is missing
+  - name: Ensure 'find' is missing from Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_without_executable_find.name }}'
+      command: 'find'
+    ignore_errors: true
+    register: _result
+
+  - name: Validate that 'find' executable is missing from Pod
+    assert:
+      that:
+      - _result is failed
+      fail_msg: "Pod contains 'find' executable, therefore we cannot run the next tasks."
+
+  - name: Copy files into container
+    k8s_cp:
+      namespace: "{{ copy_namespace }}"
+      pod: '{{ pod_without_executable_find.name }}'
+      remote_path: '{{ item.path }}'
+      content: '{{ item.content }}'
+      state: to_pod
+    with_items:
+    - path: /ansible/root.txt
+      content: this file is located at the root directory
+    - path: /ansible/.hidden_root.txt
+      content: this hidden file is located at the root directory
+    - path: /ansible/.sudir/root.txt
+      content: this file is located at the root of the sub directory
+    - path: /ansible/.sudir/.hidden_root.txt
+      content: this hidden file is located at the root of the sub directory
+
+  - name: Delete existing directory
+    file:
+      path: /tmp/openjdk-files
+      state: absent
+    ignore_errors: true
+
+  - name: copy directory from Pod into local filesystem (new directory to create)
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_without_executable_find.name }}'
+      remote_path: /ansible
+      local_path: /tmp/openjdk-files
+      state: from_pod
+
+  - name: Compare directories
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_without_executable_find.name }}'
+      remote_path: /ansible
+      local_path: /tmp/openjdk-files
+
   always:
   - name: Remove directories created into remote Pod
     k8s_exec:
@@ -79,3 +132,4 @@
     with_items:
     - /tmp/data
     - /tmp/test
+    - /tmp/openjdk-files

tests/integration/targets/k8s_copy/templates/pods_definition.j2

@@ -30,4 +30,16 @@ spec:
       - /bin/sh
       - -c
       - while true;do date;sleep 5; done
-
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: '{{ pod_without_executable_find.name }}'
+spec:
+  containers:
+  - name: openjdk17
+    image: openjdk:17
+    command:
+    - /bin/sh
+    - -c
+    - while true;do date;sleep 5; done