doc fragments workaround for downstream (#59)

* doc fragments workaround for downstream

Signed-off-by: Adam Miller <admiller@redhat.com>

* make shellcheck happy

Signed-off-by: Adam Miller <admiller@redhat.com>

* fix collection location for downstream doc fragment resolution

Signed-off-by: Adam Miller <admiller@redhat.com>

* do things in the correct order

Signed-off-by: Adam Miller <admiller@redhat.com>

* add ANSIBLE_COLLECTIONS_PATH to ansible-doc for downstream

Signed-off-by: Adam Miller <admiller@redhat.com>

* remove elements of the json dump that aren't valid for DOCUMENTATION

Signed-off-by: Adam Miller <admiller@redhat.com>

* Add fix for Ansible 2.9

Signed-off-by: Abhijeet Kasurde <akasurde@redhat.com>

* Update ci/downstream.sh

Co-authored-by: Abhijeet Kasurde <akasurde@redhat.com>
Co-authored-by: Jeff Geerling <geerlingguy@mac.com>

.github/workflows/ansible-test.yml

@@ -49,7 +49,7 @@ jobs:
 
       # run ansible-test sanity inside of Docker.
       # The docker container has all the pinned dependencies that are required.
-      # Explicity specify the version of Python we want to test
+      # Explicitly specify the version of Python we want to test
       - name: Run sanity tests
         run: make upstream-test-sanity TEST_ARGS='--python ${{ matrix.python }}'
         working-directory: ./ansible_collections/community/okd
@@ -135,7 +135,7 @@ jobs:
 
       # run ansible-test sanity inside of Docker.
       # The docker container has all the pinned dependencies that are required.
-      # Explicity specify the version of Python we want to test
+      # Explicitly specify the version of Python we want to test
       - name: Run sanity tests
         run: make downstream-test-sanity
         working-directory: ./ansible_collections/community/okd

ci/downstream.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eu
+#!/bin/bash -e
 
 # Script to dual-home the upstream and downstream Collection in a single repo
 #
@@ -44,7 +44,7 @@ f_text_sub()
     sed -i.bak "s/^version\:.*$/version: ${DOWNSTREAM_VERSION}/" "${_build_dir}/galaxy.yml"
     sed -i.bak "/STARTREMOVE/,/ENDREMOVE/d" "${_build_dir}/README.md"
 
-    find ${_build_dir} -type f -exec sed -i.bak "s/community\.okd/redhat\.openshift/g" {} \;
+    find "${_build_dir}" -type f -exec sed -i.bak "s/community\.okd/redhat\.openshift/g" {} \;
     find "${_build_dir}" -type f -name "*.bak" -delete
 }
 
@@ -76,9 +76,18 @@ f_prep()
         tests
     )
 
+    # Modules with inherited doc fragments from kubernetes.core that need
+    # rendering to deal with Galaxy/AH lack of functionality.
+    _doc_fragment_modules=(
+        k8s
+        openshift_process
+        openshift_route
+    )
 
-    # Temp build dir 
+
+    # Temp build dir
     _tmp_dir=$(mktemp -d)
+    _start_dir="${PWD}"
     _build_dir="${_tmp_dir}/ansible_collections/redhat/openshift"
     mkdir -p "${_build_dir}"
 }
@@ -127,9 +136,50 @@ f_create_collection_dir_structure()
     fi
 }
 
+f_handle_doc_fragments_workaround()
+{
+    f_log_info "${FUNCNAME[0]}"
+    local install_collections_dir="${_build_dir}/collections/"
+    local temp_fragments_json="${_tmp_dir}/fragments.json"
+    local temp_start="${_tmp_dir}/startfile.txt"
+    local temp_end="${_tmp_dir}/endfile.txt"
+    local rendered_fragments="./rendereddocfragments.txt"
+
+    # Build the collection, export docs, render them, stitch it all back together
+    pushd "${_build_dir}" || return
+        ansible-galaxy collection build
+        ansible-galaxy collection install -p "${install_collections_dir}" ./*.tar.gz
+        rm ./*.tar.gz
+        for doc_fragment_mod in "${_doc_fragment_modules[@]}"
+        do
+            local module_py="plugins/modules/${doc_fragment_mod}.py"
+            f_log_info "Processing doc fragments for ${module_py}"
+            ANSIBLE_COLLECTIONS_PATH="${install_collections_dir}" \
+            ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATH}:${install_collections_dir}" \
+                ansible-doc -j "redhat.openshift.${doc_fragment_mod}" > "${temp_fragments_json}"
+            # FIXME: Check Python interpreter from environment variable to work with prow
+            if [ -e /usr/bin/python3.6 ]; then
+                PYTHON="/usr/bin/python3.6"
+            else
+                PYTHON="python"
+            fi
+            "${PYTHON}" "${_start_dir}/ci/downstream_fragments.py" "redhat.openshift.${doc_fragment_mod}" "${temp_fragments_json}"
+            sed -n '/STARTREMOVE/q;p' "${module_py}" > "${temp_start}"
+            sed '0,/ENDREMOVE/d' "${module_py}" > "${temp_end}"
+            cat "${temp_start}" "${rendered_fragments}" "${temp_end}" > "${module_py}"
+            cat "${module_py}"
+        done
+        rm -f "${rendered_fragments}"
+        rm -fr "${install_collections_dir}"
+    popd
+
+}
+
 f_install_kubernetes_core()
 {
+    f_log_info "${FUNCNAME[0]}"
     local install_collections_dir="${_tmp_dir}/ansible_collections"
+
     pushd "${_tmp_dir}"
         ansible-galaxy collection install -p "${install_collections_dir}" kubernetes.core
     popd
@@ -149,6 +199,7 @@ f_common_steps()
     f_prep
     f_create_collection_dir_structure
     f_text_sub
+    f_handle_doc_fragments_workaround
 }
 
 # Run the test sanity scanerio
@@ -192,16 +243,16 @@ f_build_option()
 {
     f_log_info "${FUNCNAME[0]}"
     f_common_steps
-    pushd "${_build_dir}" || return 
+    pushd "${_build_dir}" || return
         f_log_info "BUILD WD: ${PWD}"
-        # FIXME 
+        # FIXME
-        #   This doesn't work because we end up either recursively curl'ing 
+        #   This doesn't work because we end up either recursively curl'ing
         #   kubernetes.core and redoing the text replacement over and over
-        #         
+        #
-        # make build 
+        # make build
         ansible-galaxy collection build
 
-    popd || return 
+    popd || return
     f_copy_collection_to_working_dir
     f_cleanup
 }
@@ -216,19 +267,19 @@ fi
 while getopts ":sirb" option
 do
   case $option in
-    s) 
+    s)
         f_test_sanity_option
         ;;
-    i) 
+    i)
         f_test_integration_option
         ;;
-    r) 
+    r)
         f_release_option
         ;;
-    b) 
+    b)
         f_build_option
         ;;
-    *) 
+    *)
         printf "ERROR: Unimplemented option chosen.\n"
         f_show_help
         f_exit 1

ci/downstream_fragments.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+import json
+import yaml
+import sys
+
+with open("./rendereddocfragments.txt", 'w') as df_fd:
+    with open(sys.argv[2], 'r') as fd:
+        json_docs = json.load(fd)
+
+        json_docs[sys.argv[1]]['doc'].pop('collection', '')
+        json_docs[sys.argv[1]]['doc'].pop('filename', '')
+
+        df_fd.write("DOCUMENTATION = '''\n")
+        df_fd.write(yaml.dump(json_docs[sys.argv[1]]['doc'], default_flow_style=False))
+        df_fd.write("'''\n\n")
+
+        df_fd.write("EXAMPLES = '''\n")
+        df_fd.write(json_docs[sys.argv[1]]['examples'])
+        df_fd.write("'''\n\n")
+
+        df_fd.write("RETURN = '''\n")
+        df_fd.write(yaml.dump(json_docs[sys.argv[1]]['return'], default_flow_style=False))
+        df_fd.write("'''\n\n")
+
+

plugins/modules/k8s.py

@@ -6,15 +6,9 @@
 
 
 from __future__ import absolute_import, division, print_function
-
-
 __metaclass__ = type
 
-
+# STARTREMOVE (downstream)
-ANSIBLE_METADATA = {'metadata_version': '1.1',
-                    'status': ['preview'],
-                    'supported_by': 'community'}
-
 DOCUMENTATION = '''
 
 module: k8s
@@ -112,8 +106,8 @@ options:
         description:
         - The value of the reason field in your desired condition
         - For example, if a C(Deployment) is paused, The C(Progressing) C(type) will have the C(DeploymentPaused) reason.
-        - The possible reasons in a condition are specific to each resource type in Kubernetes. See the API documentation of the status field
+        - The possible reasons in a condition are specific to each resource type in Kubernetes.
-          for a given resource to see possible choices.
+          See the API documentation of the status field for a given resource to see possible choices.
     type: dict
   validate:
     description:
@@ -264,6 +258,7 @@ result:
        type: int
        sample: 48
 '''
+# ENDREMOVE (downstream)
 
 import re
 import operator
@@ -396,7 +391,9 @@ class OKDRawModule(KubernetesRawModule):
                         image = existing['spec']['template']['spec']['containers'][old_container_index]['image']
                         definition['spec']['template']['spec']['containers'][new_container_index]['image'] = image
 
-                    existing_index = self.get_index(trigger['imageChangeParams'], [x.get('imageChangeParams') for x in existing_triggers], ['containerNames'])
+                    existing_index = self.get_index(trigger['imageChangeParams'],
+                                                    [x.get('imageChangeParams') for x in existing_triggers],
+                                                    ['containerNames'])
                     if existing_index is not None:
                         existing_image = existing_triggers[existing_index].get('imageChangeParams', {}).get('lastTriggeredImage')
                         if existing_image:

plugins/modules/openshift_auth.py

@@ -5,10 +5,8 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import absolute_import, division, print_function
-
 __metaclass__ = type
 
-
 DOCUMENTATION = r'''
 
 module: openshift_auth

plugins/modules/openshift_process.py

@@ -5,10 +5,9 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import absolute_import, division, print_function
-
 __metaclass__ = type
 
-
+# STARTREMOVE (downstream)
 DOCUMENTATION = r'''
 module: openshift_process
 
@@ -40,7 +39,7 @@ options:
       - The name of the Template to process.
       - The Template must be present in the cluster.
       - When provided, I(namespace) is required.
-      - Mutually exlusive with I(resource_definition) or I(src)
+      - Mutually exclusive with I(resource_definition) or I(src)
     type: str
   namespace:
     description:
@@ -202,6 +201,7 @@ resources:
              type: complex
              description: Array of status conditions for the object. Not guaranteed to be present
 '''
+# ENDREMOVE (downstream)
 
 import re
 import os

plugins/modules/openshift_route.py

@@ -8,7 +8,7 @@ from __future__ import absolute_import, division, print_function
 
 __metaclass__ = type
 
-
+# STARTREMOVE (downstream)
 DOCUMENTATION = r'''
 module: openshift_route
 
@@ -296,6 +296,7 @@ duration:
   type: int
   sample: 48
 '''
+# ENDREMOVE (downstream)
 
 import copy
 import traceback