k8s_cp - a new module for copying files to/from a Pod (#127)

* k8s_cp module

* add documentation for k8s_cp module

* add doc for the new module

* pods should be running

* support for binary, archive and zip file

* sanity

* Delete file.txt

* remove unused

* set back

* Update collection.txt

* Update test_copy_errors.yml

* Update plugins/modules/k8s_cp.py

Co-authored-by: Abhijeet Kasurde <akasurde@redhat.com>

* Update k8s_cp.py

* Update k8s_cp.py

* tar binary requirements

* Update common.py

* Update k8s_cp.py

* Update k8s_cp.py

* replace kind with binary file

* Update test_copy_large_file.yml

* Update plugins/action/k8s_info.py

Co-authored-by: Mike Graves <mgraves@redhat.com>

* Update k8s_info.py

* Update k8s_info.py

* Update k8s_cp.py

Co-authored-by: Abhijeet Kasurde <akasurde@redhat.com>
Co-authored-by: Mike Graves <mgraves@redhat.com>

molecule/default/roles/k8scopy/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+- block:
+    - name: Download kubeclt executable used to compare results
+      get_url:
+        url: https://dl.k8s.io/release/v1.21.3/bin/linux/amd64/kubectl
+        dest: "{{ kubectl_path }}"
+
+    - name: make kubectl executable
+      ansible.builtin.file:
+        path: "{{ kubectl_path }}"
+        mode: "+x"
+
+    # Ensure namespace and create pod to perform tests on
+    - name: Ensure namespace exists
+      k8s:
+        definition:
+          apiVersion: v1
+          kind: Namespace
+          metadata:
+            name: "{{ copy_namespace }}"
+
+    - name: Create Pods
+      k8s:
+        namespace: '{{ copy_namespace }}'
+        wait: yes
+        template: pods_definition.j2
+
+    - include_tasks: test_copy_errors.yml
+    - include_tasks: test_copy_file.yml
+    - include_tasks: test_multi_container_pod.yml
+    - include_tasks: test_copy_directory.yml
+    - include_tasks: test_copy_large_file.yml
+
+  always:
+    - name: Remove kubectl executable
+      ansible.builtin.file:
+        path: "{{ kubectl_path }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove namespace
+      k8s:
+        kind: Namespace
+        name: "{{ copy_namespace }}"
+        state: absent
+      ignore_errors: true

molecule/default/roles/k8scopy/tasks/test_copy_directory.yml

@@ -0,0 +1,85 @@
+---
+- block:
+  - name: copy directory into remote Pod (create new directory)
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /dest_data
+      local_path: files/data
+      state: to_pod
+
+  - name: compare directories
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /dest_data
+      local_path: '{{ role_path }}/files/data'
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: copy directory into remote Pod (existing directory)
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp
+      local_path: files/data
+      state: to_pod
+
+  - name: compare directories
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/data
+      local_path: '{{ role_path }}/files/data'
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: copy directory from Pod into local filesystem (new directory to create)
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/data
+      local_path: /tmp/test
+      state: from_pod
+
+  - name: compare directories
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/data
+      local_path: /tmp/test
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: copy directory from Pod into local filesystem (existing directory)
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/data
+      local_path: /tmp
+      state: from_pod
+
+  - name: compare directories
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/data
+      local_path: /tmp/data
+      kubectl_path: "{{ kubectl_path }}"
+
+  always:
+  - name: Remove directories created into remote Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: 'rm -rf {{ item }}'
+    ignore_errors: true
+    with_items:
+    - /dest_data
+    - /tmp/data
+
+  - name: Remove local directories
+    file:
+      path: '{{ item }}'
+      state: absent
+    ignore_errors: true
+    with_items:
+    - /tmp/data
+    - /tmp/test

molecule/default/roles/k8scopy/tasks/test_copy_errors.yml

@@ -0,0 +1,69 @@
+---
+# copy non-existent local file should fail
+- name: copy non-existent file into remote Pod
+  k8s_cp:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_one_container.name }}'
+    remote_path: /tmp
+    local_path: this_file_does_not_exist
+    state: to_pod
+  ignore_errors: true
+  register: copy_non_existent
+
+- name: check that error message is as expected
+  assert:
+    that:
+      - copy_non_existent is failed
+      - copy_non_existent.msg == "this_file_does_not_exist does not exist in local filesystem"
+
+# copy non-existent pod file should fail
+- name: copy of non-existent file from remote pod should fail
+  k8s_cp:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_one_container.name }}'
+    remote_path: /this_file_does_not_exist
+    local_path: /tmp
+    state: from_pod
+  ignore_errors: true
+  register: copy_non_existent
+
+- name: check that error message is as expected
+  assert:
+    that:
+      - copy_non_existent is failed
+      - copy_non_existent.msg == "/this_file_does_not_exist does not exist in remote pod filesystem"
+
+# copy file into multiple container pod without specifying the container should fail
+- name: copy file into multiple container pod
+  k8s_cp:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /tmp
+    local_path: files/simple_file.txt
+    state: to_pod
+  ignore_errors: true
+  register: copy_multi_container
+
+- name: check that error message is as expected
+  assert:
+    that:
+      - copy_multi_container is failed
+      - copy_multi_container.msg == "Pod contains more than 1 container, option 'container' should be set"
+
+# copy using non-existent container from pod should failed
+- name: copy file into multiple container pod
+  k8s_cp:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /tmp
+    local_path: files/simple_file.txt
+    state: to_pod
+    container: this_is_a_fake_container
+  ignore_errors: true
+  register: copy_fake_container
+
+- name: check that error message is as expected
+  assert:
+    that:
+      - copy_fake_container is failed
+      - copy_fake_container.msg == "Pod has no container this_is_a_fake_container"

molecule/default/roles/k8scopy/tasks/test_copy_file.yml

@@ -0,0 +1,191 @@
+---
+- block:
+  # Text file
+  - name: copy text file into remote pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp
+      local_path: files/simple_file.txt
+      state: to_pod
+
+  - name: Compare files
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/simple_file.txt
+      content: "{{ lookup('file', 'simple_file.txt')}}"
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: Copy simple text file from Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/simple_file.txt
+      local_path: /tmp/copy_from_pod.txt
+      state: from_pod
+
+  - name: Compare files
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/simple_file.txt
+      local_path: /tmp/copy_from_pod.txt
+      kubectl_path: "{{ kubectl_path }}"
+
+  # Binary file
+  - name: Generate random content
+    set_fact:
+      hello_arg: "{{ lookup('password', '/dev/null chars=ascii_lowercase,digits length=16') }}"
+
+  - name: Copy executable into Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/hello.exe
+      local_path: files/hello
+      state: to_pod
+
+  - name: Compare executable
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/hello.exe
+      local_path: "{{ role_path }}/files/hello"
+      kubectl_path: "{{ kubectl_path }}"
+      args:
+      - "{{ hello_arg }}"
+
+  - name: Copy executable from Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/hello.exe
+      local_path: /tmp/hello
+      state: from_pod
+
+  - name: update executable permission
+    file:
+      path: /tmp/hello
+      mode: '0755'
+
+  - name: Compare executable
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/hello.exe
+      local_path: /tmp/hello
+      kubectl_path: "{{ kubectl_path }}"
+      args:
+      - "{{ hello_arg }}"
+
+  # zip files
+  - name: copy zip file into remote pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp
+      local_path: files/simple_zip_file.txt.gz
+      state: to_pod
+
+  - name: compare zip files
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/simple_zip_file.txt.gz
+      local_path: '{{ role_path }}/files/simple_zip_file.txt.gz'
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: copy zip file from pod into local filesystem
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/simple_zip_file.txt.gz
+      local_path: /tmp/copied_from_pod.txt.gz
+      state: from_pod
+
+  - name: compare zip files
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/simple_zip_file.txt.gz
+      local_path: /tmp/copied_from_pod.txt.gz
+      kubectl_path: "{{ kubectl_path }}"
+
+  # tar files
+  - name: copy archive into remote pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp
+      local_path: files/archive.tar
+      state: to_pod
+
+  - name: compare archive
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/archive.tar
+      local_path: '{{ role_path }}/files/archive.tar'
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: copy archive from remote pod into local filesystem
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/archive.tar
+      local_path: /tmp/local_archive.tar
+      state: from_pod
+
+  - name: compare archive
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /tmp/archive.tar
+      local_path: /tmp/local_archive.tar
+      kubectl_path: "{{ kubectl_path }}"
+
+  # Copy into Pod using content option
+  - name: set content to be copied into Pod
+    set_fact:
+      pod_content: "{{ lookup('password', '/dev/null chars=ascii_lowercase,digits,punctuation length=128') }}"
+
+  - name: copy archive into remote pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /this_content.txt
+      content: '{{ pod_content }}'
+      state: to_pod
+
+  - name: Assert that content is as expected into Pod
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /this_content.txt
+      content: '{{ pod_content }}'
+      kubectl_path: "{{ kubectl_path }}"
+
+  always:
+  - name: Delete file created on Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: 'rm {{ item }}'
+    ignore_errors: true
+    with_items:
+    - /tmp/simple_file.txt
+    - /tmp/hello.exe
+    - /tmp/simple_zip_file.txt.gz
+    - /tmp/archive.tar
+    - /this_content.txt
+
+  - name: Delete file created locally
+    file:
+      path: '{{ item }}'
+      state: absent
+    with_items:
+    - /tmp/copy_from_pod.txt
+    - /tmp/hello
+    - /tmp/copied_from_pod.txt.gz
+    - /tmp/local_archive.tar

molecule/default/roles/k8scopy/tasks/test_copy_large_file.yml

@@ -0,0 +1,103 @@
+---
+- name: test copy of large binary and text files
+  block:
+  - set_fact:
+      test_directory: "/tmp/test_k8scp_large_files"
+    no_log: true
+
+  - name: create temporary directory for local files
+    ansible.builtin.file:
+      path: "{{ test_directory }}"
+      state: directory
+
+  - name: create large text file
+    k8s_create_file:
+      path: "{{ test_directory }}/large_text_file.txt"
+      size: 150
+
+  - name: create large binary file
+    k8s_create_file:
+      path: "{{ test_directory }}/large_bin_file.bin"
+      size: 200
+      binary: true
+
+  # Copy large text file from/to local filesystem to Pod
+  - name: copy large file into remote Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_text_file.txt
+      local_path: "{{ test_directory }}/large_text_file.txt"
+      state: to_pod
+
+  - name: Compare files
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_text_file.txt
+      local_path: "{{ test_directory }}/large_text_file.txt"
+      kubectl_path: "{{ kubectl_path }}"
+
+  - name: copy large file from Pod into local filesystem
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_text_file.txt
+      local_path: "{{ test_directory }}/large_text_file_from_pod.txt"
+      state: from_pod
+
+  - name: Compare files
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_text_file.txt
+      local_path: "{{ test_directory }}/large_text_file_from_pod.txt"
+      kubectl_path: "{{ kubectl_path }}"
+
+  # Copy large binary file from/to local filesystem to Pod
+  - name: copy large file into remote Pod
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_bin_file.bin
+      local_path: "{{ test_directory }}/large_bin_file.bin"
+      state: to_pod
+
+  - name: Compare executable, local vs remote
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_bin_file.bin
+      local_path: "{{ test_directory }}/large_bin_file.bin"
+
+  - name: copy executable from pod into local filesystem
+    k8s_cp:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_bin_file.bin
+      local_path: "{{ test_directory }}/large_bin_file_from_pod.bin"
+      state: from_pod
+
+  - name: Compare executable, local vs remote
+    kubectl_file_compare:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      remote_path: /large_bin_file.bin
+      local_path: "{{ test_directory }}/large_bin_file_from_pod.bin"
+
+  always:
+  - name: Delete temporary directory created for the test
+    ansible.builtin.file:
+      path: "{{ test_directory }}"
+      state: absent
+    ignore_errors: true
+
+  - name: Delete file created on Pod
+    k8s_exec:
+      namespace: '{{ copy_namespace }}'
+      pod: '{{ pod_with_one_container.name }}'
+      command: 'rm {{ item }}'
+    ignore_errors: true
+    with_items:
+    - /large_text_file.txt
+    - /large_bin_file.bin

molecule/default/roles/k8scopy/tasks/test_multi_container_pod.yml

@@ -0,0 +1,71 @@
+---
+- set_fact:
+    random_content: "{{ lookup('password', '/dev/null chars=ascii_lowercase,digits,punctuation length=128') }}"
+
+- name: Copy content into first pod's container
+  k8s_cp:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /file_from_localhost.txt
+    content: '{{ random_content }}'
+    container: '{{ pod_with_two_container.container[0] }}'
+    state: to_pod
+
+- name: Assert that content has been copied into first container
+  kubectl_file_compare:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /file_from_localhost.txt
+    container: '{{ pod_with_two_container.container[0] }}'
+    content: '{{ random_content }}'
+    kubectl_path: "{{ kubectl_path }}"
+
+- name: Assert that content has not been copied into second container
+  kubectl_file_compare:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /file_from_localhost.txt
+    container: '{{ pod_with_two_container.container[1] }}'
+    content: '{{ random_content }}'
+    kubectl_path: "{{ kubectl_path }}"
+  register: diff
+  ignore_errors: true
+
+- name: check that diff failed
+  assert:
+    that:
+      - diff is failed
+
+- name: Copy content into second's pod container
+  k8s_cp:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /file_from_localhost_01.txt
+    content: '{{ random_content }}-secondpod'
+    container: '{{ pod_with_two_container.container[1] }}'
+    state: to_pod
+
+- name: Assert that content has not been copied into first container
+  kubectl_file_compare:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /file_from_localhost_01.txt
+    container: '{{ pod_with_two_container.container[0] }}'
+    content: '{{ random_content }}-secondpod'
+    kubectl_path: "{{ kubectl_path }}"
+  ignore_errors: true
+  register: diff_1
+
+- name: check that diff failed
+  assert:
+    that:
+      - diff_1 is failed
+
+- name: Assert that content has been copied into second container
+  kubectl_file_compare:
+    namespace: '{{ copy_namespace }}'
+    pod: '{{ pod_with_two_container.name }}'
+    remote_path: /file_from_localhost_01.txt
+    container: '{{ pod_with_two_container.container[1] }}'
+    content: '{{ random_content }}-secondpod'
+    kubectl_path: "{{ kubectl_path }}"