diff --git a/changelogs/fragments/512-k8s_cp-add-support-for-check_mode-update-command-for-listing-files-into-pod.yaml b/changelogs/fragments/512-k8s_cp-add-support-for-check_mode-update-command-for-listing-files-into-pod.yaml new file mode 100644 index 00000000..a5e07f91 --- /dev/null +++ b/changelogs/fragments/512-k8s_cp-add-support-for-check_mode-update-command-for-listing-files-into-pod.yaml @@ -0,0 +1,5 @@ +--- +bugfixes: +- k8s_cp - add support for check_mode (https://github.com/ansible-collections/kubernetes.core/issues/380). +minor_changes: +- k8s_cp - remove dependency with 'find' executable on remote pod when state=from_pod (https://github.com/ansible-collections/kubernetes.core/issues/486). diff --git a/plugins/module_utils/copy.py b/plugins/module_utils/copy.py index 3a335f6b..fcea3ec8 100644 --- a/plugins/module_utils/copy.py +++ b/plugins/module_utils/copy.py @@ -64,6 +64,51 @@ class K8SCopy(metaclass=ABCMeta): self.container_arg = {} if module.params.get("container"): self.container_arg["container"] = module.params.get("container") + self.check_mode = self.module.check_mode + + def _run_from_pod(self, cmd): + try: + resp = stream( + self.api_instance.connect_get_namespaced_pod_exec, + self.name, + self.namespace, + command=cmd, + async_req=False, + stderr=True, + stdin=False, + stdout=True, + tty=False, + _preload_content=False, + **self.container_arg, + ) + + stderr, stdout = [], [] + while resp.is_open(): + resp.update(timeout=1) + if resp.peek_stdout(): + stdout.extend(resp.read_stdout().rstrip("\n").split("\n")) + if resp.peek_stderr(): + stderr.extend(resp.read_stderr().rstrip("\n").split("\n")) + error = resp.read_channel(ERROR_CHANNEL) + resp.close() + error = yaml.safe_load(error) + return error, stdout, stderr + except Exception as e: + self.module.fail_json( + msg="Error while running/parsing from pod {1}/{2} command='{0}' : {3}".format( + self.namespace, self.name, cmd, to_native(e) + ) + ) + + def is_directory_path_from_pod(self, file_path, failed_if_not_exists=True): + # check if file exists + error, out, err = self._run_from_pod(cmd=["test", "-e", file_path]) + if error.get("status") != "Success": + if failed_if_not_exists: + return None, "%s does not exist in remote pod filesystem" % file_path + return False, None + error, out, err = self._run_from_pod(cmd=["test", "-d", file_path]) + return error.get("status") == "Success", None @abstractmethod def run(self): @@ -78,56 +123,74 @@ class K8SCopyFromPod(K8SCopy): def __init__(self, module, client): super(K8SCopyFromPod, self).__init__(module, client) self.is_remote_path_dir = None - self.files_to_copy = list() + self.files_to_copy = [] + self._shellname = None + + @property + def pod_shell(self): + if self._shellname is None: + for s in ("/bin/sh", "/bin/bash"): + error, out, err = self._run_from_pod(s) + if error.get("status") == "Success": + self._shellname = s + break + return self._shellname + + def listfiles_with_find(self, path): + find_cmd = ["find", path, "-type", "f"] + error, files, err = self._run_from_pod(cmd=find_cmd) + if error.get("status") != "Success": + self.module.fail_json(msg=error.get("message")) + return files + + def listfile_with_echo(self, path): + echo_cmd = [self.pod_shell, "-c", "echo {path}/* {path}/.*".format(path=path)] + error, out, err = self._run_from_pod(cmd=echo_cmd) + if error.get("status") != "Success": + self.module.fail_json(msg=error.get("message")) + + files = [] + if out: + output = out[0] + " " + files = [ + os.path.join(path, p[:-1]) + for p in output.split(f"{path}/") + if p and p[:-1] not in (".", "..") + ] + + result = [] + for f in files: + is_dir, err = self.is_directory_path_from_pod(f) + if err: + continue + if not is_dir: + result.append(f) + continue + result += self.listfile_with_echo(f) + return result def list_remote_files(self): """ This method will check if the remote path is a dir or file if it is a directory the file list will be updated accordingly """ - try: - find_cmd = ["find", self.remote_path, "-type", "f", "-name", "*"] - response = stream( - self.api_instance.connect_get_namespaced_pod_exec, - self.name, - self.namespace, - command=find_cmd, - stdout=True, - stderr=True, - stdin=False, - tty=False, - _preload_content=False, - **self.container_arg - ) - except Exception as e: - self.module.fail_json( - msg="Failed to execute on pod {0}/{1} due to : {2}".format( - self.namespace, self.name, to_native(e) - ) - ) - stderr = [] - while response.is_open(): - response.update(timeout=1) - if response.peek_stdout(): - self.files_to_copy.extend( - response.read_stdout().rstrip("\n").split("\n") - ) - if response.peek_stderr(): - err = response.read_stderr() - if "No such file or directory" in err: - self.module.fail_json( - msg="{0} does not exist in remote pod filesystem".format( - self.remote_path - ) - ) - stderr.append(err) - error = response.read_channel(ERROR_CHANNEL) - response.close() - error = yaml.safe_load(error) - if error["status"] != "Success": - self.module.fail_json( - msg="Failed to execute on Pod due to: {0}".format(error) + # check is remote path exists and is a file or directory + is_dir, error = self.is_directory_path_from_pod(self.remote_path) + if error: + self.module.fail_json(msg=error) + + if not is_dir: + return [self.remote_path] + else: + # find executable to list dir with + executables = dict( + find=self.listfiles_with_find, + echo=self.listfile_with_echo, ) + for item in executables: + error, out, err = self._run_from_pod(item) + if error.get("status") == "Success": + return executables.get(item)(self.remote_path) def read(self): self.stdout = None @@ -162,40 +225,42 @@ class K8SCopyFromPod(K8SCopy): if is_remote_path_dir and os.path.isdir(self.local_path): relpath_start = os.path.dirname(self.remote_path) - for remote_file in self.files_to_copy: - dest_file = self.local_path - if is_remote_path_dir: - dest_file = os.path.join( - self.local_path, os.path.relpath(remote_file, start=relpath_start) - ) - # create directory to copy file in - os.makedirs(os.path.dirname(dest_file), exist_ok=True) + if not self.check_mode: + for remote_file in self.files_to_copy: + dest_file = self.local_path + if is_remote_path_dir: + dest_file = os.path.join( + self.local_path, + os.path.relpath(remote_file, start=relpath_start), + ) + # create directory to copy file in + os.makedirs(os.path.dirname(dest_file), exist_ok=True) - pod_command = ["cat", remote_file] - self.response = stream( - self.api_instance.connect_get_namespaced_pod_exec, - self.name, - self.namespace, - command=pod_command, - stderr=True, - stdin=True, - stdout=True, - tty=False, - _preload_content=False, - **self.container_arg - ) - errors = [] - with open(dest_file, "wb") as fh: - while self.response._connected: - self.read() - if self.stdout: - fh.write(self.stdout) - if self.stderr: - errors.append(self.stderr) - if errors: - self.module.fail_json( - msg="Failed to copy file from Pod: {0}".format("".join(errors)) + pod_command = ["cat", remote_file] + self.response = stream( + self.api_instance.connect_get_namespaced_pod_exec, + self.name, + self.namespace, + command=pod_command, + stderr=True, + stdin=True, + stdout=True, + tty=False, + _preload_content=False, + **self.container_arg, ) + errors = [] + with open(dest_file, "wb") as fh: + while self.response._connected: + self.read() + if self.stdout: + fh.write(self.stdout) + if self.stderr: + errors.append(self.stderr) + if errors: + self.module.fail_json( + msg="Failed to copy file from Pod: {0}".format("".join(errors)) + ) self.module.exit_json( changed=True, result="{0} successfully copied locally into {1}".format( @@ -204,7 +269,7 @@ class K8SCopyFromPod(K8SCopy): ) def run(self): - self.list_remote_files() + self.files_to_copy = self.list_remote_files() if self.files_to_copy == []: self.module.exit_json( changed=False, @@ -224,56 +289,6 @@ class K8SCopyToPod(K8SCopy): super(K8SCopyToPod, self).__init__(module, client) self.files_to_copy = list() - def run_from_pod(self, command): - response = stream( - self.api_instance.connect_get_namespaced_pod_exec, - self.name, - self.namespace, - command=command, - stderr=True, - stdin=False, - stdout=True, - tty=False, - _preload_content=False, - **self.container_arg - ) - errors = [] - while response.is_open(): - response.update(timeout=1) - if response.peek_stderr(): - errors.append(response.read_stderr()) - response.close() - err = response.read_channel(ERROR_CHANNEL) - err = yaml.safe_load(err) - response.close() - if err["status"] != "Success": - self.module.fail_json( - msg="Failed to run {0} on Pod.".format(command), errors=errors - ) - - def is_remote_path_dir(self): - pod_command = ["test", "-d", self.remote_path] - response = stream( - self.api_instance.connect_get_namespaced_pod_exec, - self.name, - self.namespace, - command=pod_command, - stdout=True, - stderr=True, - stdin=False, - tty=False, - _preload_content=False, - **self.container_arg - ) - while response.is_open(): - response.update(timeout=1) - err = response.read_channel(ERROR_CHANNEL) - err = yaml.safe_load(err) - response.close() - if err["status"] == "Success": - return True - return False - def close_temp_file(self): if self.named_temp_file: self.named_temp_file.close() @@ -296,7 +311,12 @@ class K8SCopyToPod(K8SCopy): if not os.access(self.local_path, os.R_OK): self.module.fail_json(msg="{0} not readable".format(self.local_path)) - if self.is_remote_path_dir(): + is_dir, err = self.is_directory_path_from_pod( + self.remote_path, failed_if_not_exists=False + ) + if err: + self.module.fail_json(msg=err) + if is_dir: if self.content: self.module.fail_json( msg="When content is specified, remote path should not be an existing directory" @@ -304,66 +324,67 @@ class K8SCopyToPod(K8SCopy): else: dest_file = os.path.join(dest_file, os.path.basename(src_file)) - if self.no_preserve: - tar_command = [ - "tar", - "--no-same-permissions", - "--no-same-owner", - "-xmf", - "-", - ] - else: - tar_command = ["tar", "-xmf", "-"] + if not self.check_mode: + if self.no_preserve: + tar_command = [ + "tar", + "--no-same-permissions", + "--no-same-owner", + "-xmf", + "-", + ] + else: + tar_command = ["tar", "-xmf", "-"] - if dest_file.startswith("/"): - tar_command.extend(["-C", "/"]) + if dest_file.startswith("/"): + tar_command.extend(["-C", "/"]) - response = stream( - self.api_instance.connect_get_namespaced_pod_exec, - self.name, - self.namespace, - command=tar_command, - stderr=True, - stdin=True, - stdout=True, - tty=False, - _preload_content=False, - **self.container_arg - ) - with TemporaryFile() as tar_buffer: - with tarfile.open(fileobj=tar_buffer, mode="w") as tar: - tar.add(src_file, dest_file) - tar_buffer.seek(0) - commands = [] - # push command in chunk mode - size = 1024 * 1024 - while True: - data = tar_buffer.read(size) - if not data: - break - commands.append(data) + response = stream( + self.api_instance.connect_get_namespaced_pod_exec, + self.name, + self.namespace, + command=tar_command, + stderr=True, + stdin=True, + stdout=True, + tty=False, + _preload_content=False, + **self.container_arg, + ) + with TemporaryFile() as tar_buffer: + with tarfile.open(fileobj=tar_buffer, mode="w") as tar: + tar.add(src_file, dest_file) + tar_buffer.seek(0) + commands = [] + # push command in chunk mode + size = 1024 * 1024 + while True: + data = tar_buffer.read(size) + if not data: + break + commands.append(data) - stderr, stdout = [], [] - while response.is_open(): - if response.peek_stdout(): - stdout.append(response.read_stdout().rstrip("\n")) - if response.peek_stderr(): - stderr.append(response.read_stderr().rstrip("\n")) - if commands: - cmd = commands.pop(0) - response.write_stdin(cmd) - else: - break - response.close() - if stderr: - self.close_temp_file() - self.module.fail_json( - command=tar_command, - msg="Failed to copy local file/directory into Pod due to: {0}".format( - "".join(stderr) - ), - ) - self.close_temp_file() + stderr, stdout = [], [] + while response.is_open(): + if response.peek_stdout(): + stdout.append(response.read_stdout().rstrip("\n")) + if response.peek_stderr(): + stderr.append(response.read_stderr().rstrip("\n")) + if commands: + cmd = commands.pop(0) + response.write_stdin(cmd) + else: + break + response.close() + if stderr: + self.close_temp_file() + self.module.fail_json( + command=tar_command, + msg="Failed to copy local file/directory into Pod due to: {0}".format( + "".join(stderr) + ), + ) + self.close_temp_file() if self.content: self.module.exit_json( changed=True, diff --git a/plugins/modules/k8s_cp.py b/plugins/modules/k8s_cp.py index cd277018..e8f1dea7 100644 --- a/plugins/modules/k8s_cp.py +++ b/plugins/modules/k8s_cp.py @@ -119,7 +119,7 @@ EXAMPLES = r""" state: from_pod # copy content into a file in the remote pod -- name: Copy /tmp/foo from a remote pod to /tmp/bar locally +- name: Copy content into a file in the remote pod kubernetes.core.k8s_cp: state: to_pod namespace: some-namespace diff --git a/tests/integration/targets/k8s_copy/defaults/main.yml b/tests/integration/targets/k8s_copy/defaults/main.yml index 0f3ba550..aaf46330 100644 --- a/tests/integration/targets/k8s_copy/defaults/main.yml +++ b/tests/integration/targets/k8s_copy/defaults/main.yml @@ -11,3 +11,6 @@ pod_with_two_container: container: - container-10 - container-11 + +pod_without_executable_find: + name: openjdk-pod diff --git a/tests/integration/targets/k8s_copy/tasks/main.yml b/tests/integration/targets/k8s_copy/tasks/main.yml index 824a5ee8..021faa32 100644 --- a/tests/integration/targets/k8s_copy/tasks/main.yml +++ b/tests/integration/targets/k8s_copy/tasks/main.yml @@ -19,6 +19,7 @@ template: pods_definition.j2 - include_tasks: test_copy_errors.yml + - include_tasks: test_check_mode.yml - include_tasks: test_copy_file.yml - include_tasks: test_multi_container_pod.yml - include_tasks: test_copy_directory.yml diff --git a/tests/integration/targets/k8s_copy/tasks/test_check_mode.yml b/tests/integration/targets/k8s_copy/tasks/test_check_mode.yml new file mode 100644 index 00000000..b246047e --- /dev/null +++ b/tests/integration/targets/k8s_copy/tasks/test_check_mode.yml @@ -0,0 +1,142 @@ +--- +- name: Create temporary directory for testing + tempfile: + state: directory + suffix: ansible-k8s-copy + register: tmpdir + +- block: + # setup + - name: Create local files for testing + copy: + content: "{{ item.content }}" + dest: "{{ local_dir_path }}/{{ item.dest }}" + with_items: "{{ test_files }}" + + - name: Create directory into Pod + k8s_exec: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + command: "mkdir {{ pod_dir_path }}" + + - name: Create files into Pod + k8s_cp: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + remote_path: "{{ pod_dir_path }}/{{ item.dest }}" + content: "{{ item.content }}" + state: to_pod + with_items: "{{ test_files }}" + + # Test copy into Pod using check_mode=true + - name: Copy text file into remote pod + k8s_cp: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + remote_path: "{{ pod_dir_path }}/ansible.txt" + local_path: "{{ local_dir_path }}/{{ test_files[0].dest }}" + state: to_pod + check_mode: true + register: copy_file + + - name: Ensure task is changed + assert: + that: + - copy_file is changed + + - name: Ensure file does not exists into Pod + k8s_exec: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + command: "test -e {{ pod_dir_path }}/ansible.txt" + register: test_file + failed_when: test_file.return_code == 0 + + - name: Copy directory into Pod + k8s_cp: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + remote_path: "{{ pod_dir_path }}/mydir" + local_path: "{{ local_dir_path }}" + state: to_pod + check_mode: true + register: copy_dir + + - name: Ensure task is changed + assert: + that: + - copy_dir is changed + + - name: Ensure file does not exists into Pod + k8s_exec: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + command: "test -e {{ pod_dir_path }}/mydir" + register: test_dir + failed_when: test_dir.return_code == 0 + + # Test copy from pod using check_mode=true + - name: Copy file from Pod into local file system + k8s_cp: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + remote_path: "{{ pod_dir_path }}/{{ test_files[0].dest }}" + local_path: "{{ local_dir_path }}/ansible.txt" + state: from_pod + check_mode: true + register: copy_file + + - name: Ensure task is changed + assert: + that: + - copy_file is changed + + - name: Ensure file does not exists into local file system + stat: + path: "{{ local_dir_path }}/ansible.txt" + register: testfile + failed_when: testfile.stat.exists + + - name: Copy directory from Pod into local file system + k8s_cp: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + remote_path: "{{ pod_dir_path }}" + local_path: "{{ local_dir_path }}/mydir" + state: from_pod + check_mode: true + register: _dir + + - name: Ensure task is changed + assert: + that: + - _dir is changed + + - name: Ensure directory does not exist into local file system + stat: + path: "{{ local_dir_path }}/mydir" + register: testdir + failed_when: testdir.stat.exists + + vars: + local_dir_path: "{{ tmpdir.path }}" + pod_dir_path: "/tmp/test_check_mode" + test_files: + - content: "collection = kubernetes.core" + dest: collection.txt + - content: "modules = k8s_cp and k8s_exec" + dest: modules.txt + + always: + - name: Delete temporary directory + file: + state: absent + path: "{{ local_dir_path }}" + ignore_errors: true + + - name: Delete temporary directory created into Pod + k8s_exec: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_with_one_container.name }}' + command: 'rm -r {{ pod_dir_path }}' + ignore_errors: true diff --git a/tests/integration/targets/k8s_copy/tasks/test_copy_directory.yml b/tests/integration/targets/k8s_copy/tasks/test_copy_directory.yml index fbc2418f..ed2ad85d 100644 --- a/tests/integration/targets/k8s_copy/tasks/test_copy_directory.yml +++ b/tests/integration/targets/k8s_copy/tasks/test_copy_directory.yml @@ -60,6 +60,59 @@ remote_path: /tmp/data local_path: /tmp/data + # Test copy from Pod where the executable 'find' is missing + - name: Ensure 'find' is missing from Pod + k8s_exec: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_without_executable_find.name }}' + command: 'find' + ignore_errors: true + register: _result + + - name: Validate that 'find' executable is missing from Pod + assert: + that: + - _result is failed + fail_msg: "Pod contains 'find' executable, therefore we cannot run the next tasks." + + - name: Copy files into container + k8s_cp: + namespace: "{{ copy_namespace }}" + pod: '{{ pod_without_executable_find.name }}' + remote_path: '{{ item.path }}' + content: '{{ item.content }}' + state: to_pod + with_items: + - path: /ansible/root.txt + content: this file is located at the root directory + - path: /ansible/.hidden_root.txt + content: this hidden file is located at the root directory + - path: /ansible/.sudir/root.txt + content: this file is located at the root of the sub directory + - path: /ansible/.sudir/.hidden_root.txt + content: this hidden file is located at the root of the sub directory + + - name: Delete existing directory + file: + path: /tmp/openjdk-files + state: absent + ignore_errors: true + + - name: copy directory from Pod into local filesystem (new directory to create) + k8s_cp: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_without_executable_find.name }}' + remote_path: /ansible + local_path: /tmp/openjdk-files + state: from_pod + + - name: Compare directories + kubectl_file_compare: + namespace: '{{ copy_namespace }}' + pod: '{{ pod_without_executable_find.name }}' + remote_path: /ansible + local_path: /tmp/openjdk-files + always: - name: Remove directories created into remote Pod k8s_exec: @@ -79,3 +132,4 @@ with_items: - /tmp/data - /tmp/test + - /tmp/openjdk-files diff --git a/tests/integration/targets/k8s_copy/templates/pods_definition.j2 b/tests/integration/targets/k8s_copy/templates/pods_definition.j2 index b5780543..ee6c6f65 100644 --- a/tests/integration/targets/k8s_copy/templates/pods_definition.j2 +++ b/tests/integration/targets/k8s_copy/templates/pods_definition.j2 @@ -30,4 +30,16 @@ spec: - /bin/sh - -c - while true;do date;sleep 5; done - +--- +apiVersion: v1 +kind: Pod +metadata: + name: '{{ pod_without_executable_find.name }}' +spec: + containers: + - name: openjdk17 + image: openjdk:17 + command: + - /bin/sh + - -c + - while true;do date;sleep 5; done \ No newline at end of file