diff --git a/ci/doc_fragment_modules.py b/ci/doc_fragment_modules.py new file mode 100755 index 0000000..b92abe5 --- /dev/null +++ b/ci/doc_fragment_modules.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import ast +from pathlib import PosixPath +import yaml +import argparse +import os + + +def read_docstring(filename): + + """ + Search for assignment of the DOCUMENTATION and EXAMPLES variables in the given file. + Parse DOCUMENTATION from YAML and return the YAML doc or None together with EXAMPLES, as plain text. + """ + + data = { + 'doc': None, + 'plainexamples': None, + 'returndocs': None, + 'metadata': None, # NOTE: not used anymore, kept for compat + 'seealso': None, + } + + string_to_vars = { + 'DOCUMENTATION': 'doc', + 'EXAMPLES': 'plainexamples', + 'RETURN': 'returndocs', + 'ANSIBLE_METADATA': 'metadata', # NOTE: now unused, but kept for backwards compat + } + + try: + with open(filename, 'rb') as b_module_data: + M = ast.parse(b_module_data.read()) + + for child in M.body: + if isinstance(child, ast.Assign): + for t in child.targets: + try: + theid = t.id + except AttributeError: + # skip errors can happen when trying to use the normal code + # sys.stderr.write("Failed to assign id for %s on %s, skipping\n" % (t, filename)) + continue + + if theid in string_to_vars: + varkey = string_to_vars[theid] + if isinstance(child.value, ast.Dict): + data[varkey] = ast.literal_eval(child.value) + else: + if theid != 'EXAMPLES': + # string should be yaml if already not a dict + data[varkey] = child.value.s + + # sys.stderr.write('assigned: %s\n' % varkey) + + except Exception: + # sys.stderr.write("unable to parse %s" % filename) + return + + return yaml.safe_load(data["doc"]) if data["doc"] is not None else None + + +def is_extending_collection(result, col_fqcn): + if result: + for x in result.get("extends_documentation_fragment", []): + if x.startswith(col_fqcn): + return True + return False + + +def main(): + + parser = argparse.ArgumentParser( + description="list modules with inherited doc fragments from kubernetes.core that need rendering to deal with Galaxy/AH lack of functionality." + ) + parser.add_argument( + "-c", "--collection-path", type=str, default=os.getcwd(), help="path to the collection" + ) + + args = parser.parse_args() + + path = PosixPath(args.collection_path) / PosixPath("plugins/modules") + output = [] + for d in path.iterdir(): + if d.is_file(): + result = read_docstring(str(d)) + if is_extending_collection(result, "kubernetes.core."): + output.append(d.stem.replace(".py", "")) + print("\n".join(output)) + + +if __name__ == '__main__': + main() diff --git a/ci/downstream.sh b/ci/downstream.sh index eda0d6c..ee7727d 100755 --- a/ci/downstream.sh +++ b/ci/downstream.sh @@ -11,6 +11,7 @@ DOWNSTREAM_VERSION="2.1.0" KEEP_DOWNSTREAM_TMPDIR="${KEEP_DOWNSTREAM_TMPDIR:-''}" +INSTALL_DOWNSTREAM_COLLECTION_PATH="${INSTALL_DOWNSTREAM_COLLECTION_PATH:-}" _build_dir="" f_log_info() @@ -69,6 +70,7 @@ f_prep() .yamllint requirements.txt requirements.yml + test-requirements.txt ) # Directories to recursively copy downstream (relative repo root dir path) @@ -81,15 +83,6 @@ f_prep() tests ) - # Modules with inherited doc fragments from kubernetes.core that need - # rendering to deal with Galaxy/AH lack of functionality. - _doc_fragment_modules=( - k8s - openshift_process - openshift_route - ) - - # Temp build dir _tmp_dir=$(mktemp -d) _start_dir="${PWD}" @@ -150,6 +143,15 @@ f_handle_doc_fragments_workaround() local temp_end="${_tmp_dir}/endfile.txt" local rendered_fragments="./rendereddocfragments.txt" + # FIXME: Check Python interpreter from environment variable to work with prow + PYTHON=${DOWNSTREAM_BUILD_PYTHON:-/usr/bin/python3.6} + f_log_info "Using Python interpreter: ${PYTHON}" + + # Modules with inherited doc fragments from kubernetes.core that need + # rendering to deal with Galaxy/AH lack of functionality. + # shellcheck disable=SC2207 + _doc_fragment_modules=($("${PYTHON}" "${_start_dir}/ci/doc_fragment_modules.py" -c "${_start_dir}")) + # Build the collection, export docs, render them, stitch it all back together pushd "${_build_dir}" || return ansible-galaxy collection build @@ -164,12 +166,6 @@ f_handle_doc_fragments_workaround() ANSIBLE_COLLECTIONS_PATH="${install_collections_dir}" \ ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATH}:${install_collections_dir}" \ ansible-doc -j "redhat.openshift.${doc_fragment_mod}" > "${temp_fragments_json}" - # FIXME: Check Python interpreter from environment variable to work with prow - if [ -e /usr/bin/python3.6 ]; then - PYTHON="/usr/bin/python3.6" - else - PYTHON="python" - fi "${PYTHON}" "${_start_dir}/ci/downstream_fragments.py" "redhat.openshift.${doc_fragment_mod}" "${temp_fragments_json}" sed -n '/STARTREMOVE/q;p' "${module_py}" > "${temp_start}" sed '1,/ENDREMOVE/d' "${module_py}" > "${temp_end}" @@ -185,7 +181,14 @@ f_copy_collection_to_working_dir() { f_log_info "${FUNCNAME[0]}" # Copy the Collection build result into original working dir + f_log_info "copying built collection *.tar.gz into ./" cp "${_build_dir}"/*.tar.gz ./ + # Install downstream collection into provided path + if [[ -n ${INSTALL_DOWNSTREAM_COLLECTION_PATH} ]]; then + f_log_info "Install built collection *.tar.gz into ${INSTALL_DOWNSTREAM_COLLECTION_PATH}" + ansible-galaxy collection install -p "${INSTALL_DOWNSTREAM_COLLECTION_PATH}" "${_build_dir}"/*.tar.gz + fi + rm -f "${_build_dir}"/*.tar.gz } f_common_steps() diff --git a/ci/downstream_fragments.py b/ci/downstream_fragments.py index 7fb3631..727be61 100755 --- a/ci/downstream_fragments.py +++ b/ci/downstream_fragments.py @@ -15,14 +15,18 @@ with open("./rendereddocfragments.txt", 'w') as df_fd: json_docs[sys.argv[1]]['doc'].pop('filename', '') json_docs[sys.argv[1]]['doc'].pop('has_action', '') - df_fd.write("DOCUMENTATION = '''\n") + df_fd.write('DOCUMENTATION = """\n') df_fd.write(yaml.dump(json_docs[sys.argv[1]]['doc'], default_flow_style=False)) - df_fd.write("'''\n\n") + df_fd.write('"""\n\n') - df_fd.write("EXAMPLES = '''\n") + df_fd.write('EXAMPLES = """') df_fd.write(json_docs[sys.argv[1]]['examples']) - df_fd.write("'''\n\n") + df_fd.write('"""\n\n') - df_fd.write("RETURN = '''\n") - df_fd.write(yaml.dump(json_docs[sys.argv[1]]['return'], default_flow_style=False)) - df_fd.write("'''\n\n") + df_fd.write('RETURN = r"""') + data = json_docs[sys.argv[1]]['return'] + if isinstance(data, dict): + df_fd.write(yaml.dump(data, default_flow_style=False)) + else: + df_fd.write(data) + df_fd.write('"""\n\n') diff --git a/plugins/modules/openshift_adm_groups_sync.py b/plugins/modules/openshift_adm_groups_sync.py index 3b3aff2..26fa2e2 100644 --- a/plugins/modules/openshift_adm_groups_sync.py +++ b/plugins/modules/openshift_adm_groups_sync.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +# STARTREMOVE (downstream) DOCUMENTATION = r""" module: openshift_adm_groups_sync @@ -186,6 +187,7 @@ builds: } ] """ +# ENDREMOVE (downstream) import copy import traceback diff --git a/plugins/modules/openshift_adm_migrate_template_instances.py b/plugins/modules/openshift_adm_migrate_template_instances.py index 8058c0e..2576577 100644 --- a/plugins/modules/openshift_adm_migrate_template_instances.py +++ b/plugins/modules/openshift_adm_migrate_template_instances.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type - +# STARTREMOVE (downstream) DOCUMENTATION = r""" module: openshift_adm_migrate_template_instances short_description: Update TemplateInstances to point to the latest group-version-kinds @@ -231,7 +231,7 @@ result: ... ] """ - +# ENDREMOVE (downstream) import traceback diff --git a/plugins/modules/openshift_adm_prune_auth.py b/plugins/modules/openshift_adm_prune_auth.py index 158aa22..7abd098 100644 --- a/plugins/modules/openshift_adm_prune_auth.py +++ b/plugins/modules/openshift_adm_prune_auth.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +# STARTREMOVE (downstream) DOCUMENTATION = r''' module: openshift_adm_prune_auth @@ -96,7 +97,7 @@ group: description: list of Security Context Constraints deleted. returned: I(resource=users) ''' - +# ENDREMOVE (downstream) import copy diff --git a/plugins/modules/openshift_adm_prune_deployments.py b/plugins/modules/openshift_adm_prune_deployments.py index 94621ca..424531f 100644 --- a/plugins/modules/openshift_adm_prune_deployments.py +++ b/plugins/modules/openshift_adm_prune_deployments.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +# STARTREMOVE (downstream) DOCUMENTATION = r''' module: openshift_adm_prune_deployments @@ -64,6 +65,7 @@ replication_controllers: description: list of replication controllers candidate for pruning. returned: always ''' +# ENDREMOVE (downstream) import copy diff --git a/plugins/modules/openshift_adm_prune_images.py b/plugins/modules/openshift_adm_prune_images.py index e0a2fae..c8cf1a1 100644 --- a/plugins/modules/openshift_adm_prune_images.py +++ b/plugins/modules/openshift_adm_prune_images.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +# STARTREMOVE (downstream) DOCUMENTATION = r''' module: openshift_adm_prune_images @@ -275,6 +276,7 @@ deleted_images: ... ] ''' +# ENDREMOVE (downstream) import copy diff --git a/plugins/modules/openshift_import_image.py b/plugins/modules/openshift_import_image.py index 8be1e55..10a6b22 100644 --- a/plugins/modules/openshift_import_image.py +++ b/plugins/modules/openshift_import_image.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +# STARTREMOVE (downstream) DOCUMENTATION = r''' module: openshift_import_image @@ -153,6 +154,7 @@ result: returned: success type: dict ''' +# ENDREMOVE (downstream) import copy diff --git a/plugins/modules/openshift_registry_info.py b/plugins/modules/openshift_registry_info.py index ab17fb7..ede0e4a 100644 --- a/plugins/modules/openshift_registry_info.py +++ b/plugins/modules/openshift_registry_info.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +# STARTREMOVE (downstream) DOCUMENTATION = r''' module: openshift_registry_info @@ -79,6 +80,7 @@ check: returned: always type: str ''' +# ENDREMOVE (downstream) import copy