Support resource definition using manifest URL (#478)

Support resource definition using manifest URL

SUMMARY

Closes #451

ISSUE TYPE


Feature Pull Request

COMPONENT NAME

k8s
k8s_scale
k8s_service

Reviewed-by: Mike Graves <mgraves@redhat.com>
Reviewed-by: Abhijeet Kasurde <None>
Reviewed-by: Bikouo Aubin <None>

changelogs/fragments/478-add-support-for-manifest-url.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+  - k8s, k8s_scale, k8s_service - add support for resource definition as manifest via. (https://github.com/ansible-collections/kubernetes.core/issues/451).

plugins/action/k8s_info.py

@@ -359,7 +359,7 @@ class ActionModule(ActionBase):
         # find the file in the expected search path
         src = self._task.args.get("src", None)
 
-        if src:
+        if src and not src.startswith(("http://", "https://", "ftp://")):
             if remote_transport:
                 # src is on remote node
                 result.update(

plugins/doc_fragments/k8s_resource_options.py

@@ -29,6 +29,7 @@ options:
     - Reads from the local file system. To read from the Ansible controller's file system, including vaulted files, use the file lookup
       plugin or template lookup plugin, combined with the from_yaml filter, and pass the result to
       I(resource_definition). See Examples below.
+    - The URL to manifest files that can be used to create the resource. Added in version 2.4.0.
     - Mutually exclusive with I(template) in case of M(kubernetes.core.k8s) module.
     type: path
 """

plugins/module_utils/common.py

@@ -26,6 +26,7 @@ import traceback
 import sys
 import hashlib
 from datetime import datetime
+from tempfile import NamedTemporaryFile
 
 from ansible_collections.kubernetes.core.plugins.module_utils.version import (
     LooseVersion,
@@ -47,6 +48,7 @@ from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils._text import to_native, to_bytes, to_text
 from ansible.module_utils.common.dict_transformations import dict_merge
 from ansible.module_utils.parsing.convert_bool import boolean
+from ansible.module_utils.urls import Request
 
 K8S_IMP_ERR = None
 try:
@@ -342,6 +344,28 @@ def get_api_client(module=None, **kwargs):
 get_api_client._pool = {}
 
 
+def fetch_file_from_url(module, url):
+    # Download file
+    bufsize = 65536
+    file_name, file_ext = os.path.splitext(str(url.rsplit("/", 1)[1]))
+    temp_file = NamedTemporaryFile(
+        dir=module.tmpdir, prefix=file_name, suffix=file_ext, delete=False
+    )
+    module.add_cleanup_file(temp_file.name)
+    try:
+        rsp = Request().open("GET", url)
+        if not rsp:
+            module.fail_json(msg="Failure downloading %s" % url)
+        data = rsp.read(bufsize)
+        while data:
+            temp_file.write(data)
+            data = rsp.read(bufsize)
+        temp_file.close()
+    except Exception as e:
+        module.fail_json(msg="Failure downloading %s, %s" % (url, to_native(e)))
+    return temp_file.name
+
+
 class K8sAnsibleMixin(object):
     def __init__(self, module, pyyaml_required=True, *args, **kwargs):
         module.deprecate(
@@ -529,8 +553,15 @@ class K8sAnsibleMixin(object):
                     if alias in self.params:
                         self.params.pop(alias)
 
-    def load_resource_definitions(self, src):
+    def load_resource_definitions(self, src, module=None):
         """Load the requested src path"""
+        if module and (
+            src.startswith("https://")
+            or src.startswith("http://")
+            or src.startswith("ftp://")
+        ):
+            src = fetch_file_from_url(module, src)
+
         result = None
         path = os.path.normpath(src)
         if not os.path.exists(path):
@@ -745,7 +776,7 @@ class K8sAnsibleMixin(object):
 
         src = module.params.get("src")
         if src:
-            self.resource_definitions = self.load_resource_definitions(src)
+            self.resource_definitions = self.load_resource_definitions(src, module)
         try:
             self.resource_definitions = [
                 item for item in self.resource_definitions if item
@@ -853,9 +884,9 @@ class K8sAnsibleMixin(object):
         definition["apiVersion"] = resource.group_version
         metadata = definition.get("metadata", {})
         if not metadata.get("name") and not metadata.get("generateName"):
-            if self.name:
+            if hasattr(self, "name") and self.name:
                 metadata["name"] = self.name
-            elif self.generate_name:
+            elif hasattr(self, "generate_name") and self.generate_name:
                 metadata["generateName"] = self.generate_name
         if resource.namespaced and self.namespace and not metadata.get("namespace"):
             metadata["namespace"] = self.namespace

plugins/module_utils/k8s/resource.py

@@ -5,6 +5,7 @@ import os
 from typing import cast, Dict, Iterable, List, Optional, Union
 
 from ansible.module_utils.six import string_types
+from ansible.module_utils.urls import Request
 
 try:
     import yaml
@@ -53,7 +54,11 @@ def create_definitions(params: Dict) -> List[ResourceDefinition]:
         definitions = from_yaml(d)
     elif params.get("src"):
         d = cast(str, params.get("src"))
-        definitions = from_file(d)
+        if hasattr(d, "startswith") and d.startswith(("https://", "http://", "ftp://")):
+            data = Request().open("GET", d).read().decode("utf8")
+            definitions = from_yaml(data)
+        else:
+            definitions = from_file(d)
     else:
         # We'll create an empty definition and let merge_params set values
         # from the module parameters.

plugins/module_utils/k8s/service.py

@@ -305,7 +305,6 @@ class K8sService:
     def create(self, resource: Resource, definition: Dict) -> Dict:
         namespace = definition["metadata"].get("namespace")
         name = definition["metadata"].get("name")
-        results = {"changed": False, "result": {}}
 
         if self.module.check_mode and not self.client.dry_run:
             k8s_obj = _encode_stringdata(definition)
@@ -327,7 +326,7 @@ class K8sService:
                         name
                     )
                 )
-                return results
+                return dict()
             except Exception as e:
                 reason = e.body if hasattr(e, "body") else e
                 msg = "Failed to create object: {0}".format(reason)

plugins/modules/k8s_scale.py

@@ -250,7 +250,7 @@ def execute_module(client, module):
             module.exit_json(warning=warn, **return_attributes)
 
     for existing in existing_items:
-        if module.params["kind"].lower() == "job":
+        if kind.lower() == "job":
             existing_count = existing.spec.parallelism
         elif hasattr(existing.spec, "replicas"):
             existing_count = existing.spec.replicas
@@ -285,7 +285,7 @@ def execute_module(client, module):
             continue
 
         if existing_count != replicas:
-            if module.params["kind"].lower() == "job":
+            if kind.lower() == "job":
                 existing.spec.parallelism = replicas
                 result = {"changed": True}
                 if module.check_mode:

plugins/modules/k8s_service.py

@@ -169,6 +169,9 @@ from ansible_collections.kubernetes.core.plugins.module_utils.k8s.service import
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.resource import (
     create_definitions,
 )
+from ansible_collections.kubernetes.core.plugins.module_utils.k8s.runner import (
+    perform_action,
+)
 
 
 SERVICE_ARG_SPEC = {
@@ -195,7 +198,7 @@ def merge_dicts(x, y):
             if isinstance(x[k], dict) and isinstance(y[k], dict):
                 yield (k, dict(merge_dicts(x[k], y[k])))
             else:
-                yield (k, y[k])
+                yield (k, y[k] if y[k] else x[k])
         elif k in x:
             yield (k, x[k])
         else:
@@ -211,32 +214,6 @@ def argspec():
     return argument_spec
 
 
-def perform_action(svc, resource, definition, params):
-    state = params.get("state", None)
-    result = {}
-
-    existing = svc.retrieve(resource, definition)
-
-    if state == "absent":
-        result = svc.delete(resource, definition, existing)
-        result["method"] = "delete"
-    else:
-        if params.get("apply"):
-            result = svc.apply(resource, definition, existing)
-            result["method"] = "apply"
-        elif not existing:
-            result = svc.create(resource, definition)
-            result["method"] = "create"
-        elif params.get("force", False):
-            result = svc.replace(resource, definition, existing)
-            result["method"] = "replace"
-        else:
-            result = svc.update(resource, definition, existing)
-            result["method"] = "update"
-
-    return result
-
-
 def execute_module(svc):
     """Module execution"""
     module = svc.module
@@ -263,9 +240,8 @@ def execute_module(svc):
 
     # 'resource_definition:' has lower priority than module parameters
     definition = dict(merge_dicts(definitions[0], definition))
-    resource = svc.find_resource("Service", api_version, fail=True)
 
-    result = perform_action(svc, resource, definition, module.params)
+    result = perform_action(svc, definition, module.params)
 
     module.exit_json(**result)
 

tests/integration/targets/k8s_manifest_url/aliases

@@ -0,0 +1,4 @@
+k8s_service
+k8s
+k8s_scale
+time=40

tests/integration/targets/k8s_manifest_url/defaults/main.yml

@@ -0,0 +1,64 @@
+---
+test_namespace: "k8s-manifest-url"
+file_server_container_name: "nginx-server"
+file_server_published_port: 30001
+file_server_container_image: "docker.io/nginx"
+
+pod_manifest:
+  file_name: pod.yaml
+  definition: |
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: yaml-pod
+    spec:
+      containers:
+      - name: busy
+        image: busybox
+        command:
+          - /bin/sh
+          - -c
+          - while true;do date;sleep 5; done
+
+deployment_manifest:
+  file_name: deployment.yaml
+  definition: |
+    ---
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: nginx-deployment
+      labels:
+        app: nginx
+    spec:
+      replicas: 3
+      selector:
+        matchLabels:
+          app: nginx
+      template:
+        metadata:
+          labels:
+            app: nginx
+        spec:
+          containers:
+          - name: nginx
+            image: nginx
+
+service_manifest:
+  file_name: service.yaml
+  definition: |
+    ---
+    apiVersion: v1
+    kind: Service
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      ports:
+        - name: http
+          port: 80
+      selector:
+        app: nginx
+    status:
+      loadBalancer: {}

tests/integration/targets/k8s_manifest_url/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - setup_namespace

tests/integration/targets/k8s_manifest_url/tasks/main.yml

@@ -0,0 +1,132 @@
+- name: check if docker is installed
+  shell: "command -v docker"
+  register: result
+  ignore_errors: true
+
+- block:
+    - name: Check running server
+      shell:
+        cmd: >
+          docker container ps -a
+          -f name={{ file_server_container_name }}
+          --format '{{ '{{' }} .Names {{ '}}' }}'
+      register: server
+
+    - name: Create static file server using on docker
+      block:
+        - name: Create temporary directory for file to server
+          tempfile:
+            state: directory
+            suffix: .manifests
+          register: manifests_dir
+
+        - name: Update directory permissions
+          file:
+            path: "{{ manifests_dir.path }}"
+            mode: 0755
+
+        - name: Create manifests files
+          copy:
+            content: "{{ item.definition }}"
+            dest: "{{ manifests_dir.path }}/{{ item.file_name }}"
+          with_items:
+            - "{{ pod_manifest }}"
+            - "{{ deployment_manifest }}"
+            - "{{ service_manifest }}"
+
+        - name: Create static file server
+          shell:
+            cmd: >
+              docker run
+              --name {{ file_server_container_name }}
+              -p {{ file_server_published_port }}:80
+              -v {{ manifests_dir.path }}:/usr/share/nginx/html:ro
+              -d {{ file_server_container_image }}
+
+      when: server.stdout == ""
+
+    - set_fact:
+        file_server_host: "http://127.0.0.1:{{ file_server_published_port }}"
+
+    # k8s
+    - name: Create Pod using manifest URL
+      k8s:
+        namespace: "{{ test_namespace }}"
+        src: "{{ file_server_host }}/{{ pod_manifest.file_name }}"
+        wait: true
+
+    - name: Read Pod created
+      k8s_info:
+        kind: Pod
+        namespace: "{{ test_namespace }}"
+        name: "yaml-pod"
+      register: yaml_pod
+
+    - name: Ensure Pod exists
+      assert:
+        that:
+          - yaml_pod.resources | length == 1
+
+    # k8s_scale
+    - name: Create Deployment using manifest URL
+      k8s:
+        namespace: "{{ test_namespace }}"
+        src: "{{ file_server_host }}/{{ deployment_manifest.file_name }}"
+        wait: true
+
+    - name: Scale deployment using manifest URL
+      k8s_scale:
+        namespace: "{{ test_namespace }}"
+        src: "{{ file_server_host }}/{{ deployment_manifest.file_name }}"
+        replicas: 1
+        current_replicas: 3
+        wait: true
+      register: scale
+
+    - name: Read deployment
+      k8s_info:
+        kind: Deployment
+        version: apps/v1
+        namespace: "{{ test_namespace }}"
+        name: "nginx-deployment"
+      register: deployment
+
+    - name: Ensure number of replicas has been set as requested
+      assert:
+        that:
+          - scale is changed
+          - deployment.resources | length == 1
+          - deployment.resources.0.status.replicas == 1
+
+    # k8s_service
+    - name: Create service from manifest URL
+      k8s_service:
+        name: "myservice"
+        namespace: "{{ test_namespace }}"
+        src: "{{ file_server_host }}/{{ service_manifest.file_name }}"
+      register: svc
+
+    - assert:
+        that:
+          - svc is changed
+
+  always:
+    - name: Delete namespace
+      k8s:
+        kind: Namespace
+        name: "{{ test_namespace }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Delete static file server
+      shell: "docker container rm -f {{ file_server_container_name }}"
+      ignore_errors: true
+
+    - name: Delete temporary directory
+      file:
+        state: absent
+        path: "{{ manifests_dir.path }}"
+      ignore_errors: true
+      when: manifests_dir is defined
+
+  when: result.rc == 0