Release 1.9.7.

* Extension parsing: add new fallback code which uses the new cryptography API (#331)

* Add new code as fallback which re-serializes de-serialized extensions using the new cryptography API.

* Forgot Base64 encoding.

* Add extension by OID tests.

* There's one value which is different with the new code.

* Differences in CI.

* Working around older Jinjas.

* Value depends on which SAN was included.

* Force complete CI run now since cryptography 36.0.0 is out.

ci_complete

(cherry picked from commit 3f40795a98)

* Adjust tests.

Co-authored-by: Felix Fontein <felix@fontein.de>

.azure-pipelines/azure-pipelines.yml

@@ -19,6 +19,11 @@ schedules:
     branches:
       include:
         - main
+  - cron: 0 12 * * 0
+    displayName: Weekly (old stable branches)
+    always: true
+    branches:
+      include:
         - stable-*
 
 variables:
@@ -108,16 +113,12 @@ stages:
         parameters:
           testFormat: devel/linux/{0}/1
           targets:
-            - name: CentOS 6
-              test: centos6
             - name: CentOS 7
               test: centos7
-            - name: CentOS 8
-              test: centos8
-            - name: Fedora 33
-              test: fedora33
             - name: Fedora 34
               test: fedora34
+            - name: Fedora 35
+              test: fedora35
             - name: openSUSE 15 py2
               test: opensuse15py2
             - name: openSUSE 15 py3
@@ -134,6 +135,8 @@ stages:
         parameters:
           testFormat: 2.12/linux/{0}/1
           targets:
+            - name: CentOS 6
+              test: centos6
             - name: CentOS 8
               test: centos8
             - name: Fedora 33
@@ -206,8 +209,8 @@ stages:
               test: macos/11.1
             - name: RHEL 7.9
               test: rhel/7.9
-            - name: RHEL 8.4
+            - name: RHEL 8.5
-              test: rhel/8.4
+              test: rhel/8.5
             - name: FreeBSD 12.2
               test: freebsd/12.2
             - name: FreeBSD 13.0
@@ -274,7 +277,6 @@ stages:
           nameFormat: Python {0}
           testFormat: devel/cloud/{0}/1
           targets:
-            - test: 2.6
             - test: 2.7
             - test: 3.5
             - test: 3.6
@@ -291,6 +293,7 @@ stages:
           nameFormat: Python {0}
           testFormat: 2.12/cloud/{0}/1
           targets:
+            - test: 2.6
             - test: 3.9
   - stage: Cloud_2_11
     displayName: Cloud 2.11

.azure-pipelines/scripts/aggregate-coverage.sh

@@ -11,7 +11,7 @@ mkdir "${agent_temp_directory}/coverage/"
 
 options=(--venv --venv-system-site-packages --color -v)
 
-ansible-test coverage combine --export "${agent_temp_directory}/coverage/" "${options[@]}"
+ansible-test coverage combine --group-by command --export "${agent_temp_directory}/coverage/" "${options[@]}"
 
 if ansible-test coverage analyze targets generate --help >/dev/null 2>&1; then
     # Only analyze coverage if the installed version of ansible-test supports it.

.azure-pipelines/scripts/publish-codecov.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+"""
+Upload code coverage reports to codecov.io.
+Multiple coverage files from multiple languages are accepted and aggregated after upload.
+Python coverage, as well as PowerShell and Python stubs can all be uploaded.
+"""
+
+import argparse
+import dataclasses
+import pathlib
+import shutil
+import subprocess
+import tempfile
+import typing as t
+import urllib.request
+
+
+@dataclasses.dataclass(frozen=True)
+class CoverageFile:
+    name: str
+    path: pathlib.Path
+    flags: t.List[str]
+
+
+@dataclasses.dataclass(frozen=True)
+class Args:
+    dry_run: bool
+    path: pathlib.Path
+
+
+def parse_args() -> Args:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-n', '--dry-run', action='store_true')
+    parser.add_argument('path', type=pathlib.Path)
+
+    args = parser.parse_args()
+
+    # Store arguments in a typed dataclass
+    fields = dataclasses.fields(Args)
+    kwargs = {field.name: getattr(args, field.name) for field in fields}
+
+    return Args(**kwargs)
+
+
+def process_files(directory: pathlib.Path) -> t.Tuple[CoverageFile, ...]:
+    processed = []
+    for file in directory.joinpath('reports').glob('coverage*.xml'):
+        name = file.stem.replace('coverage=', '')
+
+        # Get flags from name
+        flags = name.replace('-powershell', '').split('=')  # Drop '-powershell' suffix
+        flags = [flag if not flag.startswith('stub') else flag.split('-')[0] for flag in flags]  # Remove "-01" from stub files
+
+        processed.append(CoverageFile(name, file, flags))
+
+    return tuple(processed)
+
+
+def upload_files(codecov_bin: pathlib.Path, files: t.Tuple[CoverageFile, ...], dry_run: bool = False) -> None:
+    for file in files:
+        cmd = [
+            str(codecov_bin),
+            '--name', file.name,
+            '--file', str(file.path),
+        ]
+        for flag in file.flags:
+            cmd.extend(['--flags', flag])
+
+        if dry_run:
+            print(f'DRY-RUN: Would run command: {cmd}')
+            continue
+
+        subprocess.run(cmd, check=True)
+
+
+def download_file(url: str, dest: pathlib.Path, flags: int, dry_run: bool = False) -> None:
+    if dry_run:
+        print(f'DRY-RUN: Would download {url} to {dest} and set mode to {flags:o}')
+        return
+
+    with urllib.request.urlopen(url) as resp:
+        with dest.open('w+b') as f:
+            # Read data in chunks rather than all at once
+            shutil.copyfileobj(resp, f, 64 * 1024)
+
+    dest.chmod(flags)
+
+
+def main():
+    args = parse_args()
+    url = 'https://ansible-ci-files.s3.amazonaws.com/codecov/linux/codecov'
+    with tempfile.TemporaryDirectory(prefix='codecov-') as tmpdir:
+        codecov_bin = pathlib.Path(tmpdir) / 'codecov'
+        download_file(url, codecov_bin, 0o755, args.dry_run)
+
+        files = process_files(args.path)
+        upload_files(codecov_bin, files, args.dry_run)
+
+
+if __name__ == '__main__':
+    main()

.azure-pipelines/scripts/publish-codecov.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-# Upload code coverage reports to codecov.io.
-# Multiple coverage files from multiple languages are accepted and aggregated after upload.
-# Python coverage, as well as PowerShell and Python stubs can all be uploaded.
-
-set -o pipefail -eu
-
-output_path="$1"
-
-curl --silent --show-error https://ansible-ci-files.s3.us-east-1.amazonaws.com/codecov/codecov.sh > codecov.sh
-
-for file in "${output_path}"/reports/coverage*.xml; do
-    name="${file}"
-    name="${name##*/}"  # remove path
-    name="${name##coverage=}"  # remove 'coverage=' prefix if present
-    name="${name%.xml}"  # remove '.xml' suffix
-
-    bash codecov.sh \
-        -f "${file}" \
-        -n "${name}" \
-        -X coveragepy \
-        -X gcov \
-        -X fix \
-        -X search \
-        -X xcode \
-        || echo "Failed to upload code coverage report to codecov.io: ${file}"
-done

.azure-pipelines/scripts/report-coverage.sh

@@ -12,4 +12,4 @@ if ! ansible-test --help >/dev/null 2>&1; then
     pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
 fi
 
-ansible-test coverage xml --stub --venv --venv-system-site-packages --color -v
+ansible-test coverage xml --group-by command --stub --venv --venv-system-site-packages --color -v

.azure-pipelines/templates/coverage.yml

@@ -33,7 +33,7 @@ jobs:
           summaryFileLocation: "$(outputPath)/reports/$(pipelinesCoverage).xml"
         displayName: Publish to Azure Pipelines
         condition: gt(variables.coverageFileCount, 0)
-      - bash: .azure-pipelines/scripts/publish-codecov.sh "$(outputPath)"
+      - bash: .azure-pipelines/scripts/publish-codecov.py "$(outputPath)"
         displayName: Publish to codecov.io
         condition: gt(variables.coverageFileCount, 0)
         continueOnError: true

CHANGELOG.rst

@@ -5,6 +5,58 @@ Community Crypto Release Notes
 .. contents:: Topics
 
 
+v1.9.7
+======
+
+Release Summary
+---------------
+
+Bugfix release with extra forward compatibility for newer versions of cryptography.
+
+Minor Changes
+-------------
+
+- acme_* modules - fix usage of ``fetch_url`` with changes in latest ansible-core ``devel`` branch (https://github.com/ansible-collections/community.crypto/pull/339).
+
+Bugfixes
+--------
+
+- acme_certificate - avoid passing multiple certificates to ``cryptography``'s X.509 certificate loader when ``fullchain_dest`` is used (https://github.com/ansible-collections/community.crypto/pull/324).
+- get_certificate, openssl_csr_info, x509_certificate_info - add fallback code for extension parsing that works with cryptography 36.0.0 and newer. This code re-serializes de-serialized extensions and thus can return slightly different values if the extension in the original CSR resp. certificate was not canonicalized correctly. This code is currently used as a fallback if the existing code stops working, but we will switch it to be the main code in a future release (https://github.com/ansible-collections/community.crypto/pull/331).
+- luks_device - now also runs a built-in LUKS signature cleaner on ``state=absent`` to make sure that also the secondary LUKS2 header is wiped when older versions of wipefs are used (https://github.com/ansible-collections/community.crypto/issues/326, https://github.com/ansible-collections/community.crypto/pull/327).
+- openssl_pkcs12 - use new PKCS#12 deserialization infrastructure from cryptography 36.0.0 if available (https://github.com/ansible-collections/community.crypto/pull/302).
+
+v1.9.6
+======
+
+Release Summary
+---------------
+
+Regular bugfix release.
+
+Bugfixes
+--------
+
+- cryptography backend - improve Unicode handling for Python 2 (https://github.com/ansible-collections/community.crypto/pull/313).
+
+v1.9.5
+======
+
+Release Summary
+---------------
+
+Bugfix release to fully support cryptography 35.0.0.
+
+Bugfixes
+--------
+
+- get_certificate - fix compatibility with the cryptography 35.0.0 release (https://github.com/ansible-collections/community.crypto/pull/294).
+- openssl_csr_info - fix compatibility with the cryptography 35.0.0 release (https://github.com/ansible-collections/community.crypto/pull/294).
+- openssl_csr_info - fix compatibility with the cryptography 35.0.0 release in PyOpenSSL backend (https://github.com/ansible-collections/community.crypto/pull/300).
+- openssl_pkcs12 - fix compatibility with the cryptography 35.0.0 release (https://github.com/ansible-collections/community.crypto/pull/296).
+- x509_certificate_info - fix compatibility with the cryptography 35.0.0 release (https://github.com/ansible-collections/community.crypto/pull/294).
+- x509_certificate_info - fix compatibility with the cryptography 35.0.0 release in PyOpenSSL backend (https://github.com/ansible-collections/community.crypto/pull/300).
+
 v1.9.4
 ======
 

changelogs/changelog.yaml

@@ -562,3 +562,63 @@ releases:
     - 279-acme-openssl.yml
     - 282-acme_challenge_cert_helper-error.yml
     release_date: '2021-09-28'
+  1.9.5:
+    changes:
+      bugfixes:
+      - get_certificate - fix compatibility with the cryptography 35.0.0 release (https://github.com/ansible-collections/community.crypto/pull/294).
+      - openssl_csr_info - fix compatibility with the cryptography 35.0.0 release
+        (https://github.com/ansible-collections/community.crypto/pull/294).
+      - openssl_csr_info - fix compatibility with the cryptography 35.0.0 release
+        in PyOpenSSL backend (https://github.com/ansible-collections/community.crypto/pull/300).
+      - openssl_pkcs12 - fix compatibility with the cryptography 35.0.0 release (https://github.com/ansible-collections/community.crypto/pull/296).
+      - x509_certificate_info - fix compatibility with the cryptography 35.0.0 release
+        (https://github.com/ansible-collections/community.crypto/pull/294).
+      - x509_certificate_info - fix compatibility with the cryptography 35.0.0 release
+        in PyOpenSSL backend (https://github.com/ansible-collections/community.crypto/pull/300).
+      release_summary: Bugfix release to fully support cryptography 35.0.0.
+    fragments:
+    - 1.9.5.yml
+    - 294-cryptography-35.0.0.yml
+    - 296-openssl_pkcs12-cryptography-35.yml
+    - 300-pyopenssl-cryptography-35.yml
+    release_date: '2021-10-06'
+  1.9.6:
+    changes:
+      bugfixes:
+      - cryptography backend - improve Unicode handling for Python 2 (https://github.com/ansible-collections/community.crypto/pull/313).
+      release_summary: Regular bugfix release.
+    fragments:
+    - 1.9.6.yml
+    - 313-unicode-names.yml
+    release_date: '2021-10-30'
+  1.9.7:
+    changes:
+      bugfixes:
+      - acme_certificate - avoid passing multiple certificates to ``cryptography``'s
+        X.509 certificate loader when ``fullchain_dest`` is used (https://github.com/ansible-collections/community.crypto/pull/324).
+      - get_certificate, openssl_csr_info, x509_certificate_info - add fallback code
+        for extension parsing that works with cryptography 36.0.0 and newer. This
+        code re-serializes de-serialized extensions and thus can return slightly different
+        values if the extension in the original CSR resp. certificate was not canonicalized
+        correctly. This code is currently used as a fallback if the existing code
+        stops working, but we will switch it to be the main code in a future release
+        (https://github.com/ansible-collections/community.crypto/pull/331).
+      - luks_device - now also runs a built-in LUKS signature cleaner on ``state=absent``
+        to make sure that also the secondary LUKS2 header is wiped when older versions
+        of wipefs are used (https://github.com/ansible-collections/community.crypto/issues/326,
+        https://github.com/ansible-collections/community.crypto/pull/327).
+      - openssl_pkcs12 - use new PKCS#12 deserialization infrastructure from cryptography
+        36.0.0 if available (https://github.com/ansible-collections/community.crypto/pull/302).
+      minor_changes:
+      - acme_* modules - fix usage of ``fetch_url`` with changes in latest ansible-core
+        ``devel`` branch (https://github.com/ansible-collections/community.crypto/pull/339).
+      release_summary: Bugfix release with extra forward compatibility for newer versions
+        of cryptography.
+    fragments:
+    - 1.9.7.yml
+    - 302-openssl_pkcs12-cryptography-36.0.0.yml
+    - 324-acme_certificate-fullchain.yml
+    - 327-luks_device-wipe.yml
+    - 331-cryptography-extensions.yml
+    - fetch_url-devel.yml
+    release_date: '2021-11-22'

galaxy.yml

@@ -1,6 +1,6 @@
 namespace: community
 name: crypto
-version: 1.9.4
+version: 1.9.7
 readme: README.md
 authors:
   - Ansible (github.com/ansible)

plugins/doc_fragments/module_certificate.py

@@ -457,8 +457,8 @@ options:
             - Time will always be interpreted as UTC.
             - Valid format is C([+-]timespec | ASN.1 TIME) where timespec can be an integer
               + C([w | d | h | m | s]) (e.g. C(+32w1d2h).
-            - Note that if using relative time this module is NOT idempotent.
             - If this value is not specified, the certificate will start being valid from now.
+            - Note that this value is B(not used to determine whether an existing certificate should be regenerated).
             - This is only used by the C(ownca) provider.
         type: str
         default: +0s
@@ -470,8 +470,8 @@ options:
             - Time will always be interpreted as UTC.
             - Valid format is C([+-]timespec | ASN.1 TIME) where timespec can be an integer
               + C([w | d | h | m | s]) (e.g. C(+32w1d2h).
-            - Note that if using relative time this module is NOT idempotent.
             - If this value is not specified, the certificate will stop being valid 10 years from now.
+            - Note that this value is B(not used to determine whether an existing certificate should be regenerated).
             - This is only used by the C(ownca) provider.
             - On macOS 10.15 and onwards, TLS server certificates must have a validity period of 825 days or fewer.
               Please see U(https://support.apple.com/en-us/HT210176) for more details.
@@ -548,8 +548,8 @@ options:
             - Time will always be interpreted as UTC.
             - Valid format is C([+-]timespec | ASN.1 TIME) where timespec can be an integer
               + C([w | d | h | m | s]) (e.g. C(+32w1d2h).
-            - Note that if using relative time this module is NOT idempotent.
             - If this value is not specified, the certificate will start being valid from now.
+            - Note that this value is B(not used to determine whether an existing certificate should be regenerated).
             - This is only used by the C(selfsigned) provider.
         type: str
         default: +0s
@@ -562,8 +562,8 @@ options:
             - Time will always be interpreted as UTC.
             - Valid format is C([+-]timespec | ASN.1 TIME) where timespec can be an integer
               + C([w | d | h | m | s]) (e.g. C(+32w1d2h).
-            - Note that if using relative time this module is NOT idempotent.
             - If this value is not specified, the certificate will stop being valid 10 years from now.
+            - Note that this value is B(not used to determine whether an existing certificate should be regenerated).
             - This is only used by the C(selfsigned) provider.
             - On macOS 10.15 and onwards, TLS server certificates must have a validity period of 825 days or fewer.
               Please see U(https://support.apple.com/en-us/HT210176) for more details.

plugins/module_utils/acme/acme.py

@@ -14,8 +14,9 @@ import json
 import locale
 
 from ansible.module_utils.basic import missing_required_lib
-from ansible.module_utils.urls import fetch_url
 from ansible.module_utils.common.text.converters import to_bytes
+from ansible.module_utils.urls import fetch_url
+from ansible.module_utils.six import PY3
 
 from ansible_collections.community.crypto.plugins.module_utils.acme.backend_openssl_cli import (
     OpenSSLCLIBackend,
@@ -228,9 +229,14 @@ class ACMEClient(object):
             resp, info = fetch_url(self.module, url, data=data, headers=headers, method='POST')
             _assert_fetch_url_success(self.module, resp, info)
             result = {}
+
             try:
+                # In Python 2, reading from a closed response yields a TypeError.
+                # In Python 3, read() simply returns ''
+                if PY3 and resp.closed:
+                    raise TypeError
                 content = resp.read()
-            except AttributeError:
+            except (AttributeError, TypeError):
                 content = info.pop('body', None)
 
             if content or not parse_json_result:
@@ -284,8 +290,12 @@ class ACMEClient(object):
             _assert_fetch_url_success(self.module, resp, info)
 
             try:
+                # In Python 2, reading from a closed response yields a TypeError.
+                # In Python 3, read() simply returns ''
+                if PY3 and resp.closed:
+                    raise TypeError
                 content = resp.read()
-            except AttributeError:
+            except (AttributeError, TypeError):
                 content = info.pop('body', None)
 
         # Process result

plugins/module_utils/acme/backend_cryptography.py

@@ -14,7 +14,7 @@ import datetime
 import os
 import sys
 
-from ansible.module_utils.common.text.converters import to_bytes, to_native
+from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
 
 from ansible_collections.community.crypto.plugins.module_utils.acme.backends import (
     CryptoBackend,
@@ -41,6 +41,10 @@ from ansible_collections.community.crypto.plugins.module_utils.crypto.cryptograp
     cryptography_name_to_oid,
 )
 
+from ansible_collections.community.crypto.plugins.module_utils.crypto.pem import (
+    extract_first_pem,
+)
+
 try:
     import cryptography
     import cryptography.hazmat.backends
@@ -357,6 +361,9 @@ class CryptographyBackend(CryptoBackend):
         if cert_content is None:
             return -1
 
+        # Make sure we have at most one PEM. Otherwise cryptography 36.0.0 will barf.
+        cert_content = to_bytes(extract_first_pem(to_text(cert_content)) or '')
+
         try:
             cert = cryptography.x509.load_pem_x509_certificate(cert_content, _cryptography_backend)
         except Exception as e:

plugins/module_utils/acme/errors.py

@@ -7,8 +7,8 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-from ansible.module_utils.six import binary_type
 from ansible.module_utils.common.text.converters import to_text
+from ansible.module_utils.six import binary_type, PY3
 
 
 def format_error_problem(problem, subproblem_prefix=''):
@@ -52,8 +52,12 @@ class ACMEProtocolException(ModuleFailException):
         # Try to get hold of content, if response is given and content is not provided
         if content is None and content_json is None and response is not None:
             try:
+                # In Python 2, reading from a closed response yields a TypeError.
+                # In Python 3, read() simply returns ''
+                if PY3 and response.closed:
+                    raise TypeError
                 content = response.read()
-            except AttributeError:
+            except (AttributeError, TypeError):
                 content = info.pop('body', None)
 
         # Make sure that content_json is None or a dictionary

plugins/module_utils/crypto/_obj2txt.py

@@ -20,6 +20,10 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 
+# WARNING: this function no longer works with cryptography 35.0.0 and newer!
+#          It must **ONLY** be used in compatibility code for older
+#          cryptography versions!
+
 def obj2txt(openssl_lib, openssl_ffi, obj):
     # Set to 80 on the recommendation of
     # https://www.openssl.org/docs/crypto/OBJ_nid2ln.html#return_values

plugins/module_utils/crypto/cryptography_support.py

@@ -23,12 +23,15 @@ import base64
 import binascii
 import re
 
+from distutils.version import LooseVersion
+
 from ansible.module_utils.common.text.converters import to_text, to_bytes
 from ._asn1 import serialize_asn1_string_as_der
 
 try:
     import cryptography
     from cryptography import x509
+    from cryptography.hazmat.backends import default_backend
     from cryptography.hazmat.primitives import serialization
     import ipaddress
 except ImportError:
@@ -44,6 +47,15 @@ except ImportError:
     # Error handled in the calling module.
     _load_key_and_certificates = None
 
+try:
+    # This is a separate try/except since this is only present in cryptography 36.0.0 or newer
+    from cryptography.hazmat.primitives.serialization.pkcs12 import (
+        load_pkcs12 as _load_pkcs12,
+    )
+except ImportError:
+    # Error handled in the calling module.
+    _load_pkcs12 = None
+
 from .basic import (
     CRYPTOGRAPHY_HAS_ED25519,
     CRYPTOGRAPHY_HAS_ED448,
@@ -64,60 +76,114 @@ DOTTED_OID = re.compile(r'^\d+(?:\.\d+)+$')
 
 
 def cryptography_get_extensions_from_cert(cert):
-    # Since cryptography won't give us the DER value for an extension
-    # (that is only stored for unrecognized extensions), we have to re-do
-    # the extension parsing outselves.
     result = dict()
-    backend = cert._backend
+    try:
-    x509_obj = cert._x509
+        # Since cryptography won't give us the DER value for an extension
+        # (that is only stored for unrecognized extensions), we have to re-do
+        # the extension parsing outselves.
+        backend = default_backend()
+        try:
+            # For certain old versions of cryptography, backend is a MultiBackend object,
+            # which has no _lib attribute. In that case, revert to the old approach.
+            backend._lib
+        except AttributeError:
+            backend = cert._backend
+
+        x509_obj = cert._x509
+        # With cryptography 35.0.0, we can no longer use obj2txt. Unfortunately it still does
+        # not allow to get the raw value of an extension, so we have to use this ugly hack:
+        exts = list(cert.extensions)
+
+        for i in range(backend._lib.X509_get_ext_count(x509_obj)):
+            ext = backend._lib.X509_get_ext(x509_obj, i)
+            if ext == backend._ffi.NULL:
+                continue
+            crit = backend._lib.X509_EXTENSION_get_critical(ext)
+            data = backend._lib.X509_EXTENSION_get_data(ext)
+            backend.openssl_assert(data != backend._ffi.NULL)
+            der = backend._ffi.buffer(data.data, data.length)[:]
+            entry = dict(
+                critical=(crit == 1),
+                value=base64.b64encode(der),
+            )
+            try:
+                oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            except AttributeError:
+                oid = exts[i].oid.dotted_string
+            result[oid] = entry
+
+    except Exception:
+        # In case the above method breaks, we likely have cryptography 36.0.0 or newer.
+        # Use it's public_bytes() feature in that case. We will later switch this around
+        # so that this code will be the default, but for now this will act as a fallback
+        # since it will re-serialize de-serialized data, which can be different (if the
+        # original data was not canonicalized) from what was contained in the certificate.
+        for ext in cert.extensions:
+            result[ext.oid.dotted_string] = dict(
+                critical=ext.critical,
+                value=base64.b64encode(ext.value.public_bytes()),
+            )
 
-    for i in range(backend._lib.X509_get_ext_count(x509_obj)):
-        ext = backend._lib.X509_get_ext(x509_obj, i)
-        if ext == backend._ffi.NULL:
-            continue
-        crit = backend._lib.X509_EXTENSION_get_critical(ext)
-        data = backend._lib.X509_EXTENSION_get_data(ext)
-        backend.openssl_assert(data != backend._ffi.NULL)
-        der = backend._ffi.buffer(data.data, data.length)[:]
-        entry = dict(
-            critical=(crit == 1),
-            value=base64.b64encode(der),
-        )
-        oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
-        result[oid] = entry
     return result
 
 
 def cryptography_get_extensions_from_csr(csr):
-    # Since cryptography won't give us the DER value for an extension
-    # (that is only stored for unrecognized extensions), we have to re-do
-    # the extension parsing outselves.
     result = dict()
-    backend = csr._backend
+    try:
+        # Since cryptography won't give us the DER value for an extension
+        # (that is only stored for unrecognized extensions), we have to re-do
+        # the extension parsing outselves.
+        backend = default_backend()
+        try:
+            # For certain old versions of cryptography, backend is a MultiBackend object,
+            # which has no _lib attribute. In that case, revert to the old approach.
+            backend._lib
+        except AttributeError:
+            backend = csr._backend
 
-    extensions = backend._lib.X509_REQ_get_extensions(csr._x509_req)
+        extensions = backend._lib.X509_REQ_get_extensions(csr._x509_req)
-    extensions = backend._ffi.gc(
+        extensions = backend._ffi.gc(
-        extensions,
+            extensions,
-        lambda ext: backend._lib.sk_X509_EXTENSION_pop_free(
+            lambda ext: backend._lib.sk_X509_EXTENSION_pop_free(
-            ext,
+                ext,
-            backend._ffi.addressof(backend._lib._original_lib, "X509_EXTENSION_free")
+                backend._ffi.addressof(backend._lib._original_lib, "X509_EXTENSION_free")
+            )
         )
-    )
 
-    for i in range(backend._lib.sk_X509_EXTENSION_num(extensions)):
+        # With cryptography 35.0.0, we can no longer use obj2txt. Unfortunately it still does
-        ext = backend._lib.sk_X509_EXTENSION_value(extensions, i)
+        # not allow to get the raw value of an extension, so we have to use this ugly hack:
-        if ext == backend._ffi.NULL:
+        exts = list(csr.extensions)
-            continue
+
-        crit = backend._lib.X509_EXTENSION_get_critical(ext)
+        for i in range(backend._lib.sk_X509_EXTENSION_num(extensions)):
-        data = backend._lib.X509_EXTENSION_get_data(ext)
+            ext = backend._lib.sk_X509_EXTENSION_value(extensions, i)
-        backend.openssl_assert(data != backend._ffi.NULL)
+            if ext == backend._ffi.NULL:
-        der = backend._ffi.buffer(data.data, data.length)[:]
+                continue
-        entry = dict(
+            crit = backend._lib.X509_EXTENSION_get_critical(ext)
-            critical=(crit == 1),
+            data = backend._lib.X509_EXTENSION_get_data(ext)
-            value=base64.b64encode(der),
+            backend.openssl_assert(data != backend._ffi.NULL)
-        )
+            der = backend._ffi.buffer(data.data, data.length)[:]
-        oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            entry = dict(
-        result[oid] = entry
+                critical=(crit == 1),
+                value=base64.b64encode(der),
+            )
+            try:
+                oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            except AttributeError:
+                oid = exts[i].oid.dotted_string
+            result[oid] = entry
+
+    except Exception:
+        # In case the above method breaks, we likely have cryptography 36.0.0 or newer.
+        # Use it's public_bytes() feature in that case. We will later switch this around
+        # so that this code will be the default, but for now this will act as a fallback
+        # since it will re-serialize de-serialized data, which can be different (if the
+        # original data was not canonicalized) from what was contained in the CSR.
+        for ext in csr.extensions:
+            result[ext.oid.dotted_string] = dict(
+                critical=ext.critical,
+                value=base64.b64encode(ext.value.public_bytes()),
+            )
+
     return result
 
 
@@ -280,11 +346,11 @@ def _dn_escape_value(value):
     '''
     Escape Distinguished Name's attribute value.
     '''
-    value = value.replace('\\', '\\\\')
+    value = value.replace(u'\\', u'\\\\')
-    for ch in [',', '#', '+', '<', '>', ';', '"', '=', '/']:
+    for ch in [u',', u'#', u'+', u'<', u'>', u';', u'"', u'=', u'/']:
-        value = value.replace(ch, '\\%s' % ch)
+        value = value.replace(ch, u'\\%s' % ch)
-    if value.startswith(' '):
+    if value.startswith(u' '):
-        value = r'\ ' + value[1:]
+        value = u'\\ ' + value[1:]
     return value
 
 
@@ -294,24 +360,24 @@ def cryptography_decode_name(name):
     Raises an OpenSSLObjectError if the name is not supported.
     '''
     if isinstance(name, x509.DNSName):
-        return 'DNS:{0}'.format(name.value)
+        return u'DNS:{0}'.format(name.value)
     if isinstance(name, x509.IPAddress):
         if isinstance(name.value, (ipaddress.IPv4Network, ipaddress.IPv6Network)):
-            return 'IP:{0}/{1}'.format(name.value.network_address.compressed, name.value.prefixlen)
+            return u'IP:{0}/{1}'.format(name.value.network_address.compressed, name.value.prefixlen)
-        return 'IP:{0}'.format(name.value.compressed)
+        return u'IP:{0}'.format(name.value.compressed)
     if isinstance(name, x509.RFC822Name):
-        return 'email:{0}'.format(name.value)
+        return u'email:{0}'.format(name.value)
     if isinstance(name, x509.UniformResourceIdentifier):
-        return 'URI:{0}'.format(name.value)
+        return u'URI:{0}'.format(name.value)
     if isinstance(name, x509.DirectoryName):
-        return 'dirName:' + ''.join([
+        return u'dirName:' + u''.join([
-            '/{0}={1}'.format(cryptography_oid_to_name(attribute.oid, short=True), _dn_escape_value(attribute.value))
+            u'/{0}={1}'.format(to_text(cryptography_oid_to_name(attribute.oid, short=True)), _dn_escape_value(attribute.value))
             for attribute in name.value
         ])
     if isinstance(name, x509.RegisteredID):
-        return 'RID:{0}'.format(name.value.dotted_string)
+        return u'RID:{0}'.format(name.value.dotted_string)
     if isinstance(name, x509.OtherName):
-        return 'otherName:{0};{1}'.format(name.type_id.dotted_string, _get_hex(name.value))
+        return u'otherName:{0};{1}'.format(name.type_id.dotted_string, _get_hex(name.value))
     raise OpenSSLObjectError('Cannot decode name "{0}"'.format(name))
 
 
@@ -442,17 +508,74 @@ def cryptography_serial_number_of_cert(cert):
 def parse_pkcs12(pkcs12_bytes, passphrase=None):
     '''Returns a tuple (private_key, certificate, additional_certificates, friendly_name).
     '''
-    if _load_key_and_certificates is None:
+    if _load_pkcs12 is None and _load_key_and_certificates is None:
-        raise ValueError('load_key_and_certificates() not present in the current cryptography version')
+        raise ValueError('neither load_pkcs12() nor load_key_and_certificates() present in the current cryptography version')
-    private_key, certificate, additional_certificates = _load_key_and_certificates(
+
-        pkcs12_bytes, to_bytes(passphrase) if passphrase is not None else None)
+    if passphrase is not None:
+        passphrase = to_bytes(passphrase)
+
+    # Main code for cryptography 36.0.0 and forward
+    if _load_pkcs12 is not None:
+        return _parse_pkcs12_36_0_0(pkcs12_bytes, passphrase)
+
+    if LooseVersion(cryptography.__version__) >= LooseVersion('35.0'):
+        return _parse_pkcs12_35_0_0(pkcs12_bytes, passphrase)
+
+    return _parse_pkcs12_legacy(pkcs12_bytes, passphrase)
+
+
+def _parse_pkcs12_36_0_0(pkcs12_bytes, passphrase=None):
+    # Requires cryptography 36.0.0 or newer
+    pkcs12 = _load_pkcs12(pkcs12_bytes, passphrase)
+    additional_certificates = [cert.certificate for cert in pkcs12.additional_certs]
+    private_key = pkcs12.key
+    certificate = None
+    friendly_name = None
+    if pkcs12.cert:
+        certificate = pkcs12.cert.certificate
+        friendly_name = pkcs12.cert.friendly_name
+    return private_key, certificate, additional_certificates, friendly_name
+
+
+def _parse_pkcs12_35_0_0(pkcs12_bytes, passphrase=None):
+    # Backwards compatibility code for cryptography 35.x
+    private_key, certificate, additional_certificates = _load_key_and_certificates(pkcs12_bytes, passphrase)
 
     friendly_name = None
     if certificate:
         # See https://github.com/pyca/cryptography/issues/5760#issuecomment-842687238
-        maybe_name = certificate._backend._lib.X509_alias_get0(
+        backend = default_backend()
-            certificate._x509, certificate._backend._ffi.NULL)
+
-        if maybe_name != certificate._backend._ffi.NULL:
+        # This code basically does what load_key_and_certificates() does, but without error-checking.
-            friendly_name = certificate._backend._ffi.string(maybe_name)
+        # Since load_key_and_certificates succeeded, it should not fail.
+        pkcs12 = backend._ffi.gc(
+            backend._lib.d2i_PKCS12_bio(backend._bytes_to_bio(pkcs12_bytes).bio, backend._ffi.NULL),
+            backend._lib.PKCS12_free)
+        certificate_x509_ptr = backend._ffi.new("X509 **")
+        with backend._zeroed_null_terminated_buf(to_bytes(passphrase) if passphrase is not None else None) as passphrase_buffer:
+            backend._lib.PKCS12_parse(
+                pkcs12,
+                passphrase_buffer,
+                backend._ffi.new("EVP_PKEY **"),
+                certificate_x509_ptr,
+                backend._ffi.new("Cryptography_STACK_OF_X509 **"))
+        if certificate_x509_ptr[0] != backend._ffi.NULL:
+            maybe_name = backend._lib.X509_alias_get0(certificate_x509_ptr[0], backend._ffi.NULL)
+            if maybe_name != backend._ffi.NULL:
+                friendly_name = backend._ffi.string(maybe_name)
 
     return private_key, certificate, additional_certificates, friendly_name
+
+
+def _parse_pkcs12_legacy(pkcs12_bytes, passphrase=None):
+    # Backwards compatibility code for cryptography < 35.0.0
+    private_key, certificate, additional_certificates = _load_key_and_certificates(pkcs12_bytes, passphrase)
+
+    friendly_name = None
+    if certificate:
+        # See https://github.com/pyca/cryptography/issues/5760#issuecomment-842687238
+        backend = certificate._backend
+        maybe_name = backend._lib.X509_alias_get0(certificate._x509, backend._ffi.NULL)
+        if maybe_name != backend._ffi.NULL:
+            friendly_name = backend._ffi.string(maybe_name)
+    return private_key, certificate, additional_certificates, friendly_name

plugins/module_utils/crypto/pem.py

@@ -72,3 +72,13 @@ def split_pem_list(text, keep_inbetween=False):
                     result.append(''.join(current))
                     current = [] if keep_inbetween else None
     return result
+
+
+def extract_first_pem(text):
+    '''
+    Given one PEM or multiple concatenated PEM objects, return only the first one, or None if there is none.
+    '''
+    all_pems = split_pem_list(text)
+    if not all_pems:
+        return None
+    return all_pems[0]

plugins/module_utils/crypto/pyopenssl_support.py

@@ -21,10 +21,12 @@ __metaclass__ = type
 
 import base64
 
-from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native
 
 from ansible_collections.community.crypto.plugins.module_utils.compat import ipaddress as compat_ipaddress
 
+from ._objects import OID_LOOKUP
+
 try:
     import OpenSSL
 except ImportError:
@@ -87,18 +89,25 @@ def pyopenssl_get_extensions_from_cert(cert):
             critical=bool(ext.get_critical()),
             value=base64.b64encode(ext.get_data()),
         )
-        oid = obj2txt(
+        try:
-            OpenSSL._util.lib,
+            oid = obj2txt(
-            OpenSSL._util.ffi,
+                OpenSSL._util.lib,
-            OpenSSL._util.lib.X509_EXTENSION_get_object(ext._extension)
+                OpenSSL._util.ffi,
-        )
+                OpenSSL._util.lib.X509_EXTENSION_get_object(ext._extension)
-        # This could also be done a bit simpler:
+            )
-        #
+            # This could also be done a bit simpler:
-        #   oid = obj2txt(OpenSSL._util.lib, OpenSSL._util.ffi, OpenSSL._util.lib.OBJ_nid2obj(ext._nid))
+            #
-        #
+            #   oid = obj2txt(OpenSSL._util.lib, OpenSSL._util.ffi, OpenSSL._util.lib.OBJ_nid2obj(ext._nid))
-        # Unfortunately this gives the wrong result in case the linked OpenSSL
+            #
-        # doesn't know the OID. That's why we have to get the OID dotted string
+            # Unfortunately this gives the wrong result in case the linked OpenSSL
-        # similarly to how cryptography does it.
+            # doesn't know the OID. That's why we have to get the OID dotted string
+            # similarly to how cryptography does it.
+        except AttributeError:
+            # When PyOpenSSL is used with cryptography >= 35.0.0, obj2txt cannot be used.
+            # We try to figure out the OID with our internal lookup table, and if we fail,
+            # we use the short name OpenSSL returns.
+            oid = to_native(ext.get_short_name())
+            oid = OID_LOOKUP.get(oid, oid)
         result[oid] = entry
     return result
 
@@ -113,18 +122,25 @@ def pyopenssl_get_extensions_from_csr(csr):
             critical=bool(ext.get_critical()),
             value=base64.b64encode(ext.get_data()),
         )
-        oid = obj2txt(
+        try:
-            OpenSSL._util.lib,
+            oid = obj2txt(
-            OpenSSL._util.ffi,
+                OpenSSL._util.lib,
-            OpenSSL._util.lib.X509_EXTENSION_get_object(ext._extension)
+                OpenSSL._util.ffi,
-        )
+                OpenSSL._util.lib.X509_EXTENSION_get_object(ext._extension)
-        # This could also be done a bit simpler:
+            )
-        #
+            # This could also be done a bit simpler:
-        #   oid = obj2txt(OpenSSL._util.lib, OpenSSL._util.ffi, OpenSSL._util.lib.OBJ_nid2obj(ext._nid))
+            #
-        #
+            #   oid = obj2txt(OpenSSL._util.lib, OpenSSL._util.ffi, OpenSSL._util.lib.OBJ_nid2obj(ext._nid))
-        # Unfortunately this gives the wrong result in case the linked OpenSSL
+            #
-        # doesn't know the OID. That's why we have to get the OID dotted string
+            # Unfortunately this gives the wrong result in case the linked OpenSSL
-        # similarly to how cryptography does it.
+            # doesn't know the OID. That's why we have to get the OID dotted string
+            # similarly to how cryptography does it.
+        except AttributeError:
+            # When PyOpenSSL is used with cryptography >= 35.0.0, obj2txt cannot be used.
+            # We try to figure out the OID with our internal lookup table, and if we fail,
+            # we use the short name OpenSSL returns.
+            oid = to_native(ext.get_short_name())
+            oid = OID_LOOKUP.get(oid, oid)
         result[oid] = entry
     return result
 

plugins/modules/luks_device.py

@@ -356,6 +356,34 @@ LUKS_NAME_REGEX = re.compile(r'\s*crypt\s+([^\s]*)\s*')
 LUKS_DEVICE_REGEX = re.compile(r'\s*device:\s+([^\s]*)\s*')
 
 
+# See https://gitlab.com/cryptsetup/cryptsetup/-/wikis/LUKS-standard/on-disk-format.pdf
+LUKS_HEADER = b'LUKS\xba\xbe'
+LUKS_HEADER_L = 6
+# See https://gitlab.com/cryptsetup/LUKS2-docs/-/blob/master/luks2_doc_wip.pdf
+LUKS2_HEADER_OFFSETS = [0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000]
+LUKS2_HEADER2 = b'SKUL\xba\xbe'
+
+
+def wipe_luks_headers(device):
+    wipe_offsets = []
+    with open(device, 'rb') as f:
+        # f.seek(0)
+        data = f.read(LUKS_HEADER_L)
+        if data == LUKS_HEADER:
+            wipe_offsets.append(0)
+        for offset in LUKS2_HEADER_OFFSETS:
+            f.seek(offset)
+            data = f.read(LUKS_HEADER_L)
+            if data == LUKS2_HEADER2:
+                wipe_offsets.append(offset)
+
+    if wipe_offsets:
+        with open(device, 'wb') as f:
+            for offset in wipe_offsets:
+                f.seek(offset)
+                f.write(b'\x00\x00\x00\x00\x00\x00')
+
+
 class Handler(object):
 
     def __init__(self, module):
@@ -515,9 +543,17 @@ class CryptHandler(Handler):
             self.run_luks_close(name)
         result = self._run_command([wipefs_bin, '--all', device])
         if result[RETURN_CODE] != 0:
-            raise ValueError('Error while wiping luks container %s: %s'
+            raise ValueError('Error while wiping LUKS container signatures for %s: %s'
                              % (device, result[STDERR]))
 
+        # For LUKS2, sometimes both `cryptsetup erase` and `wipefs` do **not**
+        # erase all LUKS signatures (they seem to miss the second header). That's
+        # why we do it ourselves here.
+        try:
+            wipe_luks_headers(device)
+        except Exception as exc:
+            raise ValueError('Error while wiping LUKS container signatures for %s: %s' % (device, exc))
+
     def run_luks_add_key(self, device, keyfile, passphrase, new_keyfile,
                          new_passphrase, pbkdf):
         ''' Add new key from a keyfile or passphrase to given 'device';

tests/integration/targets/openssl_csr_info/tasks/impl.yml

@@ -8,7 +8,7 @@
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
 
-- name: "({{ select_crypto_backend }}) Check whether subject behaves as expected"
+- name: "({{ select_crypto_backend }}) Check whether subject and extensions behaves as expected"
   assert:
     that:
       - result.subject.organizationalUnitName == 'ACME Department'
@@ -16,6 +16,21 @@
       - "['organizationalUnitName', 'ACME Department'] in result.subject_ordered"
       - result.public_key_type == 'RSA'
       - result.public_key_data.size == default_rsa_key_size
+      # TLS Feature
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].critical == false
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].value == 'MAMCAQU='
+      # Key Usage
+      - result.extensions_by_oid['2.5.29.15'].critical == true
+      - result.extensions_by_oid['2.5.29.15'].value in ['AwMA/4A=', 'AwMH/4A=']
+      # Subject Alternative Names
+      - result.extensions_by_oid['2.5.29.17'].critical == false
+      - result.extensions_by_oid['2.5.29.17'].value == 'MGCCD3d3dy5hbnNpYmxlLmNvbYcEAQIDBIcQAAAAAAAAAAAAAAAAAAAAAYEQdGVzdEBleGFtcGxlLm9yZ4YjaHR0cHM6Ly9leGFtcGxlLm9yZy90ZXN0L2luZGV4Lmh0bWw='
+      # Basic Constraints
+      - result.extensions_by_oid['2.5.29.19'].critical == true
+      - result.extensions_by_oid['2.5.29.19'].value == 'MAYBAf8CARc='
+      # Extended Key Usage
+      - result.extensions_by_oid['2.5.29.37'].critical == false
+      - result.extensions_by_oid['2.5.29.37'].value == 'MHQGCCsGAQUFBwMBBggrBgEFBQcDAQYIKwYBBQUHAwIGCCsGAQUFBwMDBggrBgEFBQcDBAYIKwYBBQUHAwgGCCsGAQUFBwMJBgRVHSUABggrBgEFBQcBAwYIKwYBBQUHAwoGCCsGAQUFBwMHBggrBgEFBQcBAg=='
 
 - name: "({{ select_crypto_backend }}) Check SubjectKeyIdentifier and AuthorityKeyIdentifier"
   assert:
@@ -24,6 +39,10 @@
       - result.authority_key_identifier == "44:55:66:77"
       - result.authority_cert_issuer == expected_authority_cert_issuer
       - result.authority_cert_serial_number == 12345
+      # Subject Key Identifier
+      - result.extensions_by_oid['2.5.29.14'].critical == false
+      # Authority Key Identifier
+      - result.extensions_by_oid['2.5.29.35'].critical == false
   vars:
     expected_authority_cert_issuer:
       - "DNS:ca.example.org"

tests/integration/targets/x509_certificate_info/tasks/impl.yml

@@ -8,7 +8,7 @@
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
 
-- name: Check whether issuer and subject behave as expected
+- name: Check whether issuer and subject and extensions behave as expected
   assert:
     that:
       - result.issuer.organizationalUnitName == 'ACME Department'
@@ -19,6 +19,28 @@
       - "['organizationalUnitName', 'ACME Department'] in result.subject_ordered"
       - result.public_key_type == 'RSA'
       - result.public_key_data.size == (default_rsa_key_size_certifiates | int)
+      - "result.subject_alt_name == [
+          'DNS:www.ansible.com',
+          'IP:1.2.3.4',
+          'IP:::1',
+          'email:test@example.org',
+          'URI:https://example.org/test/index.html'
+        ]"
+      # TLS Feature
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].critical == false
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].value == 'MAMCAQU='
+      # Key Usage
+      - result.extensions_by_oid['2.5.29.15'].critical == true
+      - result.extensions_by_oid['2.5.29.15'].value in ['AwMA/4A=', 'AwMH/4A=']
+      # Subject Alternative Names
+      - result.extensions_by_oid['2.5.29.17'].critical == false
+      - result.extensions_by_oid['2.5.29.17'].value == 'MGCCD3d3dy5hbnNpYmxlLmNvbYcEAQIDBIcQAAAAAAAAAAAAAAAAAAAAAYEQdGVzdEBleGFtcGxlLm9yZ4YjaHR0cHM6Ly9leGFtcGxlLm9yZy90ZXN0L2luZGV4Lmh0bWw='
+      # Basic Constraints
+      - result.extensions_by_oid['2.5.29.19'].critical == true
+      - result.extensions_by_oid['2.5.29.19'].value == 'MAYBAf8CARc='
+      # Extended Key Usage
+      - result.extensions_by_oid['2.5.29.37'].critical == false
+      - result.extensions_by_oid['2.5.29.37'].value == 'MHQGCCsGAQUFBwMBBggrBgEFBQcDAQYIKwYBBQUHAwIGCCsGAQUFBwMDBggrBgEFBQcDBAYIKwYBBQUHAwgGCCsGAQUFBwMJBgRVHSUABggrBgEFBQcBAwYIKwYBBQUHAwoGCCsGAQUFBwMHBggrBgEFBQcBAg=='
 
 - name: Check SubjectKeyIdentifier and AuthorityKeyIdentifier
   assert:
@@ -27,6 +49,10 @@
       - result.authority_key_identifier == "44:55:66:77"
       - result.authority_cert_issuer == expected_authority_cert_issuer
       - result.authority_cert_serial_number == 12345
+      # Subject Key Identifier
+      - result.extensions_by_oid['2.5.29.14'].critical == false
+      # Authority Key Identifier
+      - result.extensions_by_oid['2.5.29.35'].critical == false
   vars:
     expected_authority_cert_issuer:
       - "DNS:ca.example.org"
@@ -122,10 +148,39 @@
     path: '{{ remote_tmp_dir }}/packed-cert-1.pem'
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
-- assert:
+- name: Check extensions
+  assert:
     that:
       - "'ocsp_uri' in result"
       - "result.ocsp_uri == 'http://ocsp.int-x3.letsencrypt.org'"
+      - result.extensions_by_oid | length == 9
+      # Precert Signed Certificate Timestamps
+      - result.extensions_by_oid['1.3.6.1.4.1.11129.2.4.2'].critical == false
+      - result.extensions_by_oid['1.3.6.1.4.1.11129.2.4.2'].value == 'BIHyAPAAdgDBFkrgp3LS1DktyArBB3DU8MSb3pkaSEDB+gdRZPYzYAAAAWTdAoU6AAAEAwBHMEUCIG5WpfKF536KKa9fnVlYbwcfrKh09Hi2MSRwU2kad49UAiEA4RUKjJOgw11IHFNdit+sy1RcCU3QCSOEQYrJ1/oPltAAdgApPFGWVMg5ZbqqUPxYB9S3b79Yeily3KTDDPTlRUf0eAAAAWTdAoc+AAAEAwBHMEUCIQCJjo75K4rVDSiWQe3XFLY6MiG3zcHQrKb0YhM17r1UKAIgGa8qMoN03DLp+Rm9nRJ9XLbTJz1vbuu9PyXUY741P8E='
+      # Authority Information Access
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.1'].critical == false
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.1'].value == 'MGEwLgYIKwYBBQUHMAGGImh0dHA6Ly9vY3NwLmludC14My5sZXRzZW5jcnlwdC5vcmcwLwYIKwYBBQUHMAKGI2h0dHA6Ly9jZXJ0LmludC14My5sZXRzZW5jcnlwdC5vcmcv'
+      # Subject Key Identifier
+      - result.extensions_by_oid['2.5.29.14'].critical == false
+      - result.extensions_by_oid['2.5.29.14'].value == 'BBRtcOI/yg62Ehbu5vQzxMUUdBOYMw=='
+      # Key Usage (The certificate has 'AwIFoA==', while de-serializing and re-serializing yields 'AwIAoA=='!)
+      - result.extensions_by_oid['2.5.29.15'].critical == true
+      - result.extensions_by_oid['2.5.29.15'].value in ['AwIFoA==', 'AwIAoA==']
+      # Subject Alternative Names
+      - result.extensions_by_oid['2.5.29.17'].critical == false
+      - result.extensions_by_oid['2.5.29.17'].value == 'MIIB5IIbY2VydC5pbnQteDEubGV0c2VuY3J5cHQub3JnghtjZXJ0LmludC14Mi5sZXRzZW5jcnlwdC5vcmeCG2NlcnQuaW50LXgzLmxldHNlbmNyeXB0Lm9yZ4IbY2VydC5pbnQteDQubGV0c2VuY3J5cHQub3JnghxjZXJ0LnJvb3QteDEubGV0c2VuY3J5cHQub3Jngh9jZXJ0LnN0YWdpbmcteDEubGV0c2VuY3J5cHQub3Jngh9jZXJ0LnN0Zy1pbnQteDEubGV0c2VuY3J5cHQub3JngiBjZXJ0LnN0Zy1yb290LXgxLmxldHNlbmNyeXB0Lm9yZ4ISY3AubGV0c2VuY3J5cHQub3JnghpjcC5yb290LXgxLmxldHNlbmNyeXB0Lm9yZ4ITY3BzLmxldHNlbmNyeXB0Lm9yZ4IbY3BzLnJvb3QteDEubGV0c2VuY3J5cHQub3Jnghtjcmwucm9vdC14MS5sZXRzZW5jcnlwdC5vcmeCD2xldHNlbmNyeXB0Lm9yZ4IWb3JpZ2luLmxldHNlbmNyeXB0Lm9yZ4IXb3JpZ2luMi5sZXRzZW5jcnlwdC5vcmeCFnN0YXR1cy5sZXRzZW5jcnlwdC5vcmeCE3d3dy5sZXRzZW5jcnlwdC5vcmc='
+      # Basic Constraints
+      - result.extensions_by_oid['2.5.29.19'].critical == true
+      - result.extensions_by_oid['2.5.29.19'].value == 'MAA='
+      # Certificate Policies
+      - result.extensions_by_oid['2.5.29.32'].critical == false
+      - result.extensions_by_oid['2.5.29.32'].value == 'MIHzMAgGBmeBDAECATCB5gYLKwYBBAGC3xMBAQEwgdYwJgYIKwYBBQUHAgEWGmh0dHA6Ly9jcHMubGV0c2VuY3J5cHQub3JnMIGrBggrBgEFBQcCAjCBngyBm1RoaXMgQ2VydGlmaWNhdGUgbWF5IG9ubHkgYmUgcmVsaWVkIHVwb24gYnkgUmVseWluZyBQYXJ0aWVzIGFuZCBvbmx5IGluIGFjY29yZGFuY2Ugd2l0aCB0aGUgQ2VydGlmaWNhdGUgUG9saWN5IGZvdW5kIGF0IGh0dHBzOi8vbGV0c2VuY3J5cHQub3JnL3JlcG9zaXRvcnkv'
+      # Authority Key Identifier
+      - result.extensions_by_oid['2.5.29.35'].critical == false
+      - result.extensions_by_oid['2.5.29.35'].value == 'MBaAFKhKamMEfd265tE5t6ZFZe/zqOyh'
+      # Extended Key Usage
+      - result.extensions_by_oid['2.5.29.37'].critical == false
+      - result.extensions_by_oid['2.5.29.37'].value == 'MBQGCCsGAQUFBwMBBggrBgEFBQcDAg=='
 - name: Check fingerprints
   assert:
     that:

tests/sanity/ignore-2.10.txt

@@ -1,3 +1,9 @@
+.azure-pipelines/scripts/publish-codecov.py replace-urlopen
+.azure-pipelines/scripts/publish-codecov.py compile-2.6!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py compile-2.7!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py compile-3.5!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py future-import-boilerplate
+.azure-pipelines/scripts/publish-codecov.py metaclass-boilerplate
 plugins/module_utils/acme/__init__.py empty-init
 plugins/module_utils/compat/ipaddress.py future-import-boilerplate
 plugins/module_utils/compat/ipaddress.py metaclass-boilerplate

tests/sanity/ignore-2.11.txt

@@ -1,3 +1,9 @@
+.azure-pipelines/scripts/publish-codecov.py replace-urlopen
+.azure-pipelines/scripts/publish-codecov.py compile-2.6!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py compile-2.7!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py compile-3.5!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py future-import-boilerplate
+.azure-pipelines/scripts/publish-codecov.py metaclass-boilerplate
 plugins/module_utils/acme/__init__.py empty-init
 plugins/module_utils/compat/ipaddress.py future-import-boilerplate
 plugins/module_utils/compat/ipaddress.py metaclass-boilerplate

tests/sanity/ignore-2.12.txt

@@ -1,3 +1,4 @@
+.azure-pipelines/scripts/publish-codecov.py replace-urlopen
 plugins/module_utils/acme/__init__.py empty-init
 plugins/module_utils/compat/ipaddress.py future-import-boilerplate
 plugins/module_utils/compat/ipaddress.py metaclass-boilerplate

tests/sanity/ignore-2.13.txt

@@ -1,7 +1,7 @@
+.azure-pipelines/scripts/publish-codecov.py replace-urlopen
 plugins/module_utils/acme/__init__.py empty-init
 plugins/module_utils/compat/ipaddress.py future-import-boilerplate
 plugins/module_utils/compat/ipaddress.py metaclass-boilerplate
 plugins/module_utils/compat/ipaddress.py no-assert
 plugins/module_utils/compat/ipaddress.py no-unicode-literals
 plugins/module_utils/crypto/__init__.py empty-init
-plugins/modules/acme_account_info.py validate-modules:return-syntax-error

tests/sanity/ignore-2.9.txt

@@ -1,3 +1,9 @@
+.azure-pipelines/scripts/publish-codecov.py replace-urlopen
+.azure-pipelines/scripts/publish-codecov.py compile-2.6!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py compile-2.7!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py compile-3.5!skip # Uses Python 3.6+ syntax
+.azure-pipelines/scripts/publish-codecov.py future-import-boilerplate
+.azure-pipelines/scripts/publish-codecov.py metaclass-boilerplate
 plugins/module_utils/acme/__init__.py empty-init
 plugins/module_utils/compat/ipaddress.py future-import-boilerplate
 plugins/module_utils/compat/ipaddress.py metaclass-boilerplate

tests/unit/plugins/module_utils/acme/fixtures/csr_1.pem

@@ -1,4 +1,4 @@
------BEGIN NEW CERTIFICATE REQUEST-----
+-----BEGIN CERTIFICATE REQUEST-----
 MIIBJTCBzQIBADAWMRQwEgYDVQQDEwthbnNpYmxlLmNvbTBZMBMGByqGSM49AgEG
 CCqGSM49AwEHA0IABACc9MgAFwMBJjoU0ZI18cIHnW1juoKG2DN5VrM60uvBvEEs
 4V0egJkNyM2Q4pp001zu14VcpQ0/Ei8xOOPxKZugVTBTBgkqhkiG9w0BCQ4xRjBE
@@ -6,4 +6,4 @@ MCMGA1UdEQQcMBqCC2V4YW1wbGUuY29tggtleGFtcGxlLm9yZzAMBgNVHRMBAf8E
 AjAAMA8GA1UdDwEB/wQFAwMHgAAwCgYIKoZIzj0EAwIDRwAwRAIgcDyoRmwFVBDl
 FvbFZtiSd5wmJU1ltM6JtcfnLWnjY54CICruOByrropFUkOKKb4xXOYsgaDT93Wr
 URnCJfTLr2T3
------END NEW CERTIFICATE REQUEST-----
+-----END CERTIFICATE REQUEST-----

tests/unit/plugins/module_utils/acme/fixtures/csr_1.pem.old

@@ -0,0 +1,12 @@
+cryptography 35.0.0 does not support the 'NEW' in there; so to fix tests we removed it.
+Once cryptography is fixed we should revert to the old version.
+
+-----BEGIN NEW CERTIFICATE REQUEST-----
+MIIBJTCBzQIBADAWMRQwEgYDVQQDEwthbnNpYmxlLmNvbTBZMBMGByqGSM49AgEG
+CCqGSM49AwEHA0IABACc9MgAFwMBJjoU0ZI18cIHnW1juoKG2DN5VrM60uvBvEEs
+4V0egJkNyM2Q4pp001zu14VcpQ0/Ei8xOOPxKZugVTBTBgkqhkiG9w0BCQ4xRjBE
+MCMGA1UdEQQcMBqCC2V4YW1wbGUuY29tggtleGFtcGxlLm9yZzAMBgNVHRMBAf8E
+AjAAMA8GA1UdDwEB/wQFAwMHgAAwCgYIKoZIzj0EAwIDRwAwRAIgcDyoRmwFVBDl
+FvbFZtiSd5wmJU1ltM6JtcfnLWnjY54CICruOByrropFUkOKKb4xXOYsgaDT93Wr
+URnCJfTLr2T3
+-----END NEW CERTIFICATE REQUEST-----

tests/unit/plugins/module_utils/acme/test_errors.py

@@ -115,12 +115,14 @@ def test_format_error_problem(problem, subproblem_prefix, result):
 def create_regular_response(response_text):
     response = MagicMock()
     response.read = MagicMock(return_value=response_text.encode('utf-8'))
+    response.closed = False
     return response
 
 
 def create_error_response():
     response = MagicMock()
     response.read = MagicMock(side_effect=AttributeError('read'))
+    response.closed = True
     return response
 
 

tests/unit/plugins/modules/test_luks_device.py

@@ -58,6 +58,9 @@ def test_run_luks_remove(monkeypatch):
     monkeypatch.setattr(luks_device.Handler,
                         "_run_command",
                         run_command_check)
+    monkeypatch.setattr(luks_device,
+                        "wipe_luks_headers",
+                        lambda device: True)
     crypt = luks_device.CryptHandler(module)
     crypt.run_luks_remove("dummy")
 

tests/utils/shippable/shippable.sh

@@ -49,7 +49,7 @@ function retry
         echo "@* -> ${result}"
     done
     echo "Command '@*' failed 3 times!"
-    exit -1
+    exit 255
 }
 
 command -v pip
@@ -97,7 +97,7 @@ if [ "${test}" == "sanity/extra" ]; then
     # retry ansible-galaxy -vvv collection install community.internal_test_tools
 fi
 
-if [ "${script}" != "units" ] && [ "${script}" != "sanity" ] && [ "${ansible_version}" != "2.9" ]; then
+if [ "${script}" != "units" ] && [ "${script}" != "sanity" ] && [ "${test}" != "sanity/extra" ] && [ "${ansible_version}" != "2.9" ]; then
     retry git clone --depth=1 --single-branch https://github.com/ansible-collections/community.general.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general"
     # NOTE: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429)
     # retry ansible-galaxy -vvv collection install community.general