Add extra sanity tests (#43)

* Prepare extra sanity tests.

* Run some extra sanity tests.

* Linting.

* Experimenting

* Linting.

* Run tests in docker container.

* Improve run code.

* Use requests instead of Ansible code (which needs to be installed first).

* Refactor and add more debugging.

* More ls.

* More ls.

* More ls in container.

* Copy cwd into container.

* Improve container handling.

* Make tests configurable.

* Using proper argument parser.

* Add colors.

* Allow to disable test.

* Always run all tests for now. (Change detection needs to know about git I think).

* Use argument parser for main executable. Force colors for CI.

* Allow to specify targets.

* Linting.

* Switch to ansibulled-changelog.

* Use runner from community.internal_test_tools (current dev branch for runner).

* Clone official repository instead of my fork/branch.

changelogs/config.yaml

@@ -0,0 +1,22 @@
+changes_file: changelog.yaml
+changes_format: combined
+notesdir: fragments
+prelude_section_name: release_summary
+prelude_section_title: Release Summary
+sections:
+- - major_changes
+  - Major Changes
+- - minor_changes
+  - Minor Changes
+- - breaking_changes
+  - Breaking Changes / Porting Guide
+- - deprecated_features
+  - Deprecated Features
+- - removed_features
+  - Removed Features (previously deprecated)
+- - security_fixes
+  - Security Fixes
+- - bugfixes
+  - Bugfixes
+- - known_issues
+  - Known Issues

shippable.yml

@@ -9,6 +9,7 @@ matrix:
     - env: T=none
   include:
     - env: T=devel/sanity/1
+    - env: T=devel/sanity/extra
 
     - env: T=devel/units/2.7/1
     - env: T=devel/units/3.5/1

tests/sanity/extra/changelog.json

@@ -0,0 +1,10 @@
+{
+    "python": "3.7",
+    "output": "path-line-column-message",
+    "prefixes": [
+        "changelogs/fragments/"
+    ],
+    "requirements": [
+        "git+git://github.com/ansible-community/ansibulled.git@pip-installable#egg=ansibulled"
+    ]
+}

tests/sanity/extra/changelog.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import os
+import sys
+import subprocess
+
+
+def main():
+    paths = sys.argv[1:] or sys.stdin.read().splitlines()
+
+    allowed_extensions = ('.yml', '.yaml')
+
+    for path in paths:
+        ext = os.path.splitext(path)[1]
+
+        if ext not in allowed_extensions:
+            print('%s:%d:%d: extension must be one of: %s' % (path, 0, 0, ', '.join(allowed_extensions)))
+
+    cmd = ['ansibulled-changelog', 'lint'] + paths
+    subprocess.check_call(cmd)
+
+
+if __name__ == '__main__':
+    main()

tests/sanity/extra/no-unwanted-files.json

@@ -0,0 +1,7 @@
+{
+    "include_symlinks": true,
+    "prefixes": [
+        "plugins/"
+    ],
+    "output": "path-message"
+}

tests/sanity/extra/no-unwanted-files.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+"""Prevent unwanted files from being added to the source tree."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import sys
+
+
+def main():
+    """Main entry point."""
+    paths = sys.argv[1:] or sys.stdin.read().splitlines()
+
+    allowed_extensions = (
+        '.cs',
+        '.ps1',
+        '.psm1',
+        '.py',
+    )
+
+    skip_paths = set([
+    ])
+
+    skip_directories = (
+    )
+
+    for path in paths:
+        if path in skip_paths:
+            continue
+
+        if any(path.startswith(skip_directory) for skip_directory in skip_directories):
+            continue
+
+        ext = os.path.splitext(path)[1]
+
+        if ext not in allowed_extensions:
+            print('%s: extension must be one of: %s' % (path, ', '.join(allowed_extensions)))
+
+
+if __name__ == '__main__':
+    main()

tests/sanity/extra/update-bundled.json

@@ -0,0 +1,15 @@
+{
+    "disabled": true,
+    "all_targets": true,
+    "ignore_self": true,
+    "extensions": [
+        ".py"
+    ],
+    "prefixes": [
+        "plugins/module_utils/compat/"
+    ],
+    "output": "path-message",
+    "requirements": [
+        "requests"
+    ]
+}

tests/sanity/extra/update-bundled.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2018 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+"""
+This test checks whether the libraries we're bundling are out of date and need to be synced with
+a newer upstream release.
+"""
+
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import re
+import sys
+from distutils.version import LooseVersion
+
+import packaging.specifiers
+
+import requests
+
+
+BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b')
+
+
+def get_bundled_libs(paths):
+    """
+    Return the set of known bundled libraries
+
+    :arg paths: The paths which the test has been instructed to check
+    :returns: The list of all files which we know to contain bundled libraries.  If a bundled
+        library consists of multiple files, this should be the file which has metadata included.
+    """
+    bundled_libs = set()
+    bundled_libs.add('plugins/module_utils/compat/ipaddress.py')
+
+    return bundled_libs
+
+
+def get_files_with_bundled_metadata(paths):
+    """
+    Search for any files which have bundled metadata inside of them
+
+    :arg paths: Iterable of filenames to search for metadata inside of
+    :returns: A set of pathnames which contained metadata
+    """
+
+    with_metadata = set()
+    for path in paths:
+        with open(path, 'rb') as f:
+            body = f.read()
+
+        if BUNDLED_RE.search(body):
+            with_metadata.add(path)
+
+    return with_metadata
+
+
+def get_bundled_metadata(filename):
+    """
+    Retrieve the metadata about a bundled library from a python file
+
+    :arg filename: The filename to look inside for the metadata
+    :raises ValueError: If we're unable to extract metadata from the file
+    :returns: The metadata from the python file
+    """
+    with open(filename, 'r') as module:
+        for line in module:
+            if line.strip().startswith('_BUNDLED_METADATA'):
+                data = line[line.index('{'):].strip()
+                break
+        else:
+            raise ValueError('Unable to check bundled library for update.  Please add'
+                             ' _BUNDLED_METADATA dictionary to the library file with'
+                             ' information on pypi name and bundled version.')
+        metadata = json.loads(data)
+    return metadata
+
+
+def get_latest_applicable_version(pypi_data, constraints=None):
+    """Get the latest pypi version of the package that we allow
+
+    :arg pypi_data: Pypi information about the data as returned by
+        ``https://pypi.org/pypi/{pkg_name}/json``
+    :kwarg constraints: version constraints on what we're allowed to use as specified by
+        the bundled metadata
+    :returns: The most recent version on pypi that are allowed by ``constraints``
+    """
+    latest_version = "0"
+    if constraints:
+        version_specification = packaging.specifiers.SpecifierSet(constraints)
+        for version in pypi_data['releases']:
+            if version in version_specification:
+                if LooseVersion(version) > LooseVersion(latest_version):
+                    latest_version = version
+    else:
+        latest_version = pypi_data['info']['version']
+
+    return latest_version
+
+
+def main():
+    """Entrypoint to the script"""
+
+    paths = sys.argv[1:] or sys.stdin.read().splitlines()
+
+    bundled_libs = get_bundled_libs(paths)
+    files_with_bundled_metadata = get_files_with_bundled_metadata(paths)
+
+    for filename in files_with_bundled_metadata.difference(bundled_libs):
+        print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to'
+              ' test/sanity/code-smell/update-bundled.py'.format(filename))
+
+    for filename in bundled_libs:
+        try:
+            metadata = get_bundled_metadata(filename)
+        except ValueError as e:
+            print('{0}: ERROR: {1}'.format(filename, e))
+            continue
+        except (IOError, OSError) as e:
+            if e.errno == 2:
+                print('{0}: ERROR: {1}.  Perhaps the bundled library has been removed'
+                      ' or moved and the bundled library test needs to be modified as'
+                      ' well?'.format(filename, e))
+
+        pypi_r = requests.get('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
+        pypi_data = pypi_r.json()
+
+        constraints = metadata.get('version_constraints', None)
+        latest_version = get_latest_applicable_version(pypi_data, constraints)
+
+        if LooseVersion(metadata['version']) < LooseVersion(latest_version):
+            print('{0}: UPDATE {1} from {2} to {3} {4}'.format(
+                filename,
+                metadata['pypi_name'],
+                metadata['version'],
+                latest_version,
+                'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))
+
+
+if __name__ == '__main__':
+    main()

tests/utils/shippable/sanity.sh

@@ -2,12 +2,25 @@
 
 set -o pipefail -eux
 
+declare -a args
+IFS='/:' read -ra args <<< "$1"
+
+group="${args[1]}"
+
 if [ "${BASE_BRANCH:-}" ]; then
     base_branch="origin/${BASE_BRANCH}"
 else
     base_branch=""
 fi
 
+if [ "${group}" == "extra" ]; then
+    # ansible-galaxy -vvv collection install community.internal_test_tools
+    git clone --single-branch --depth 1 https://github.com/ansible-collections/community.internal_test_tools.git ../internal_test_tools
+
+    ../internal_test_tools/tools/run.py --color
+    exit
+fi
+
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --docker --base-branch "${base_branch}" \