diff --git a/changelogs/config.yaml b/changelogs/config.yaml new file mode 100644 index 00000000..22f7aa51 --- /dev/null +++ b/changelogs/config.yaml @@ -0,0 +1,22 @@ +changes_file: changelog.yaml +changes_format: combined +notesdir: fragments +prelude_section_name: release_summary +prelude_section_title: Release Summary +sections: +- - major_changes + - Major Changes +- - minor_changes + - Minor Changes +- - breaking_changes + - Breaking Changes / Porting Guide +- - deprecated_features + - Deprecated Features +- - removed_features + - Removed Features (previously deprecated) +- - security_fixes + - Security Fixes +- - bugfixes + - Bugfixes +- - known_issues + - Known Issues diff --git a/shippable.yml b/shippable.yml index 66b8e884..f2daf2cf 100644 --- a/shippable.yml +++ b/shippable.yml @@ -9,6 +9,7 @@ matrix: - env: T=none include: - env: T=devel/sanity/1 + - env: T=devel/sanity/extra - env: T=devel/units/2.7/1 - env: T=devel/units/3.5/1 diff --git a/tests/sanity/extra/changelog.json b/tests/sanity/extra/changelog.json new file mode 100644 index 00000000..909175d9 --- /dev/null +++ b/tests/sanity/extra/changelog.json @@ -0,0 +1,10 @@ +{ + "python": "3.7", + "output": "path-line-column-message", + "prefixes": [ + "changelogs/fragments/" + ], + "requirements": [ + "git+git://github.com/ansible-community/ansibulled.git@pip-installable#egg=ansibulled" + ] +} \ No newline at end of file diff --git a/tests/sanity/extra/changelog.py b/tests/sanity/extra/changelog.py new file mode 100644 index 00000000..243ce952 --- /dev/null +++ b/tests/sanity/extra/changelog.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +import os +import sys +import subprocess + + +def main(): + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + allowed_extensions = ('.yml', '.yaml') + + for path in paths: + ext = os.path.splitext(path)[1] + + if ext not in allowed_extensions: + print('%s:%d:%d: extension must be one of: %s' % (path, 0, 0, ', '.join(allowed_extensions))) + + cmd = ['ansibulled-changelog', 'lint'] + paths + subprocess.check_call(cmd) + + +if __name__ == '__main__': + main() diff --git a/tests/sanity/extra/no-unwanted-files.json b/tests/sanity/extra/no-unwanted-files.json new file mode 100644 index 00000000..c789a7fd --- /dev/null +++ b/tests/sanity/extra/no-unwanted-files.json @@ -0,0 +1,7 @@ +{ + "include_symlinks": true, + "prefixes": [ + "plugins/" + ], + "output": "path-message" +} diff --git a/tests/sanity/extra/no-unwanted-files.py b/tests/sanity/extra/no-unwanted-files.py new file mode 100755 index 00000000..49806f2e --- /dev/null +++ b/tests/sanity/extra/no-unwanted-files.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +"""Prevent unwanted files from being added to the source tree.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import sys + + +def main(): + """Main entry point.""" + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + allowed_extensions = ( + '.cs', + '.ps1', + '.psm1', + '.py', + ) + + skip_paths = set([ + ]) + + skip_directories = ( + ) + + for path in paths: + if path in skip_paths: + continue + + if any(path.startswith(skip_directory) for skip_directory in skip_directories): + continue + + ext = os.path.splitext(path)[1] + + if ext not in allowed_extensions: + print('%s: extension must be one of: %s' % (path, ', '.join(allowed_extensions))) + + +if __name__ == '__main__': + main() diff --git a/tests/sanity/extra/update-bundled.json b/tests/sanity/extra/update-bundled.json new file mode 100644 index 00000000..83d27fb0 --- /dev/null +++ b/tests/sanity/extra/update-bundled.json @@ -0,0 +1,15 @@ +{ + "disabled": true, + "all_targets": true, + "ignore_self": true, + "extensions": [ + ".py" + ], + "prefixes": [ + "plugins/module_utils/compat/" + ], + "output": "path-message", + "requirements": [ + "requests" + ] +} diff --git a/tests/sanity/extra/update-bundled.py b/tests/sanity/extra/update-bundled.py new file mode 100755 index 00000000..09f3c296 --- /dev/null +++ b/tests/sanity/extra/update-bundled.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +""" +This test checks whether the libraries we're bundling are out of date and need to be synced with +a newer upstream release. +""" + + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import re +import sys +from distutils.version import LooseVersion + +import packaging.specifiers + +import requests + + +BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b') + + +def get_bundled_libs(paths): + """ + Return the set of known bundled libraries + + :arg paths: The paths which the test has been instructed to check + :returns: The list of all files which we know to contain bundled libraries. If a bundled + library consists of multiple files, this should be the file which has metadata included. + """ + bundled_libs = set() + bundled_libs.add('plugins/module_utils/compat/ipaddress.py') + + return bundled_libs + + +def get_files_with_bundled_metadata(paths): + """ + Search for any files which have bundled metadata inside of them + + :arg paths: Iterable of filenames to search for metadata inside of + :returns: A set of pathnames which contained metadata + """ + + with_metadata = set() + for path in paths: + with open(path, 'rb') as f: + body = f.read() + + if BUNDLED_RE.search(body): + with_metadata.add(path) + + return with_metadata + + +def get_bundled_metadata(filename): + """ + Retrieve the metadata about a bundled library from a python file + + :arg filename: The filename to look inside for the metadata + :raises ValueError: If we're unable to extract metadata from the file + :returns: The metadata from the python file + """ + with open(filename, 'r') as module: + for line in module: + if line.strip().startswith('_BUNDLED_METADATA'): + data = line[line.index('{'):].strip() + break + else: + raise ValueError('Unable to check bundled library for update. Please add' + ' _BUNDLED_METADATA dictionary to the library file with' + ' information on pypi name and bundled version.') + metadata = json.loads(data) + return metadata + + +def get_latest_applicable_version(pypi_data, constraints=None): + """Get the latest pypi version of the package that we allow + + :arg pypi_data: Pypi information about the data as returned by + ``https://pypi.org/pypi/{pkg_name}/json`` + :kwarg constraints: version constraints on what we're allowed to use as specified by + the bundled metadata + :returns: The most recent version on pypi that are allowed by ``constraints`` + """ + latest_version = "0" + if constraints: + version_specification = packaging.specifiers.SpecifierSet(constraints) + for version in pypi_data['releases']: + if version in version_specification: + if LooseVersion(version) > LooseVersion(latest_version): + latest_version = version + else: + latest_version = pypi_data['info']['version'] + + return latest_version + + +def main(): + """Entrypoint to the script""" + + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + bundled_libs = get_bundled_libs(paths) + files_with_bundled_metadata = get_files_with_bundled_metadata(paths) + + for filename in files_with_bundled_metadata.difference(bundled_libs): + print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to' + ' test/sanity/code-smell/update-bundled.py'.format(filename)) + + for filename in bundled_libs: + try: + metadata = get_bundled_metadata(filename) + except ValueError as e: + print('{0}: ERROR: {1}'.format(filename, e)) + continue + except (IOError, OSError) as e: + if e.errno == 2: + print('{0}: ERROR: {1}. Perhaps the bundled library has been removed' + ' or moved and the bundled library test needs to be modified as' + ' well?'.format(filename, e)) + + pypi_r = requests.get('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])) + pypi_data = pypi_r.json() + + constraints = metadata.get('version_constraints', None) + latest_version = get_latest_applicable_version(pypi_data, constraints) + + if LooseVersion(metadata['version']) < LooseVersion(latest_version): + print('{0}: UPDATE {1} from {2} to {3} {4}'.format( + filename, + metadata['pypi_name'], + metadata['version'], + latest_version, + 'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))) + + +if __name__ == '__main__': + main() diff --git a/tests/utils/shippable/sanity.sh b/tests/utils/shippable/sanity.sh index dd1e68b9..c216220e 100755 --- a/tests/utils/shippable/sanity.sh +++ b/tests/utils/shippable/sanity.sh @@ -2,12 +2,25 @@ set -o pipefail -eux +declare -a args +IFS='/:' read -ra args <<< "$1" + +group="${args[1]}" + if [ "${BASE_BRANCH:-}" ]; then base_branch="origin/${BASE_BRANCH}" else base_branch="" fi +if [ "${group}" == "extra" ]; then + # ansible-galaxy -vvv collection install community.internal_test_tools + git clone --single-branch --depth 1 https://github.com/ansible-collections/community.internal_test_tools.git ../internal_test_tools + + ../internal_test_tools/tools/run.py --color + exit +fi + # shellcheck disable=SC2086 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --docker --base-branch "${base_branch}" \