upstream ci: Move Azure scripts to infra directory

Move Azure scripts to infra directory, as only roles and modules test
playbooks should exist in the tests directory.

infra/azure/azure-pipelines.yml

@@ -0,0 +1,85 @@
+---
+trigger:
+- master
+
+pool:
+  vmImage: 'ubuntu-20.04'
+
+stages:
+
+# Fedora
+
+- stage: Fedora_Ansible_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core"
+
+# Fedora
+
+- stage: Fedora_Ansible_min_supported
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core <2.14"
+
+# Fedora
+
+- stage: Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.14,<2.15"
+
+# Galaxy on Fedora
+
+- stage: Galaxy_Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/galaxy_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.14,<2.15"
+
+# CentOS 9 Stream
+
+- stage: CentOS_9_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c9s
+      ansible_version: "-core >=2.14,<2.15"
+
+# CentOS 8 Stream
+
+- stage: CentOS_8_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c8s
+      ansible_version: "-core >=2.14,<2.15"
+
+# CentOS 7
+
+- stage: CentOS_7
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: centos-7
+      ansible_version: "-core >=2.14,<2.15"

infra/azure/build-containers.yml

@@ -0,0 +1,60 @@
+---
+
+schedules:
+- cron: "0 0 * * 0"
+  displayName: Weekly Sunday midnight build
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+
+pool:
+  vmImage: 'ubuntu-22.04'
+
+stages:
+
+# Currently, it's not possible to use CentOS container
+#
+# - stage: CentOS_7
+#   dependsOn: []
+#   jobs:
+#   - template: templates/build_container.yml
+#     parameters:
+#       job_name_suffix: Centos7
+#       distro: centos-7
+
+- stage: CentOS_8_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/build_container.yml
+    parameters:
+      job_name_suffix: C8S
+      distro: c8s
+      # ansible-core 2.17+ cannot be used to deploy on CentOS 8 Stream.
+      ansible_core_version: "<2.17"
+
+- stage: CentOS_9_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/build_container.yml
+    parameters:
+      job_name_suffix: C9S
+      distro: c9s
+
+- stage: Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/build_container.yml
+    parameters:
+      job_name_suffix: FedoraLatest
+      distro: fedora-latest
+
+- stage: Fedora_Rawhide
+  dependsOn: []
+  jobs:
+  - template: templates/build_container.yml
+    parameters:
+      job_name_suffix: FedoraRawhide
+      distro: fedora-rawhide

infra/azure/nightly.yml

@@ -0,0 +1,243 @@
+---
+schedules:
+- cron: "0 19 * * *"
+  displayName: Nightly Builds
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+
+pool:
+  vmImage: 'ubuntu-20.04'
+
+stages:
+
+# Fedora
+
+- stage: FedoraLatest_Ansible_Core_2_13
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.13,<2.14"
+
+- stage: FedoraLatest_Ansible_Core_2_14
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.14,<2.15"
+
+- stage: FedoraLatest_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.15,<2.16"
+
+- stage: FedoraLatest_Ansible_latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: ""
+
+# Galaxy on Fedora
+
+- stage: Galaxy_FedoraLatest_Ansible_Core_2_13
+  dependsOn: []
+  jobs:
+  - template: templates/galaxy_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.13,<2.14"
+
+- stage: Galaxy_FedoraLatest_Ansible_Core_2_14
+  dependsOn: []
+  jobs:
+  - template: templates/galaxy_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.14,<2.15"
+
+- stage: Galaxy_FedoraLatest_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/galaxy_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.15,<2.16"
+
+- stage: Galaxy_FedoraLatest_Ansible_latest
+  dependsOn: []
+  jobs:
+  - template: templates/galaxy_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: ""
+
+# Fedora Rawhide
+
+- stage: FedoraRawhide_Ansible_Core_2_13
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-rawhide
+      ansible_version: "-core >=2.13,<2.14"
+
+- stage: FedoraRawhide_Ansible_Core_2_14
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-rawhide
+      ansible_version: "-core >=2.14,<2.15"
+
+- stage: FedoraRawhide_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-rawhide
+      ansible_version: "-core >=2.15,<2.16"
+
+- stage: FedoraRawhide_Ansible_latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-rawhide
+      ansible_version: ""
+
+# CentoOS 9 Stream
+
+- stage: c9s_Ansible_Core_2_13
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c9s
+      ansible_version: "-core >=2.13,<2.14"
+
+- stage: c9s_Ansible_Core_2_14
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c9s
+      ansible_version: "-core >=2.14,<2.15"
+
+- stage: c9s_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c9s
+      ansible_version: "-core >=2.15,<2.16"
+
+- stage: c9s_Ansible_latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c9s
+      ansible_version: ""
+
+# CentOS 8 Stream
+
+- stage: c8s_Ansible_Core_2_13
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c8s
+      ansible_version: "-core >=2.13,<2.14"
+
+- stage: c8s_Ansible_Core_2_14
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c8s
+      ansible_version: "-core >=2.14,<2.15"
+
+- stage: c8s_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c8s
+      ansible_version: "-core >=2.15,<2.16"
+
+- stage: c8s_Ansible_latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c8s
+      ansible_version: ""
+
+# CentOS 7
+
+- stage: CentOS7_Ansible_Core_2_13
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: centos-7
+      ansible_version: "-core >=2.13,<2.14"
+
+- stage: CentOS7_Ansible_Core_2_14
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: centos-7
+      ansible_version: "-core >=2.14,<2.15"
+
+- stage: CentOS7_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: centos-7
+      ansible_version: "-core >=2.15,<2.16"
+
+- stage: CentOS7_Ansible_latest
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: centos-7
+      ansible_version: ""

infra/azure/pr-pipeline.yml

@@ -0,0 +1,78 @@
+---
+trigger:
+- master
+
+pool:
+  vmImage: 'ubuntu-20.04'
+
+stages:
+
+# Fedora
+
+- stage: Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-latest
+      ansible_version: "-core >=2.15,<2.16"
+
+# Galaxy on Fedora
+
+- stage: Galaxy_Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-latest
+      ansible_version: "-core >=2.15,<2.16"
+
+# CentOS 9 Stream
+
+- stage: CentOS_9_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c9s
+      ansible_version: "-core >=2.15,<2.16"
+
+# CentOS 8 Stream
+
+- stage: CentOS_8_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c8s
+      ansible_version: "-core >=2.15,<2.16"
+      target_python: "/usr/libexec/platform-python"
+
+# CentOS 7 cannot be used with current systemd
+#
+# CentOS 7
+#
+# - stage: CentOS_7
+#   dependsOn: []
+#   jobs:
+#   - template: templates/fast_tests.yml
+#     parameters:
+#       build_number: $(Build.BuildNumber)
+#       distro: centos-7
+#       ansible_version: "-core >=2.15,<2.16"
+#       target_python: "/usr/bin/python2"
+
+# Rawhide
+
+- stage: Fedora_Rawhide
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-rawhide
+      ansible_version: "-core >=2.15,<2.16"

infra/azure/templates/build_container.yml

@@ -0,0 +1,50 @@
+---
+parameters:
+  - name: job_name_suffix
+    type: string
+  - name: distro
+    type: string
+  - name: python_version
+    type: string
+    default: 3.x
+  - name: ansible_core_version
+    default: ""
+
+jobs:
+- job: BuildTestImage${{ parameters.job_name_suffix }}
+  displayName: Build ${{ parameters.distro }} test container
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: python -m pip install --upgrade pip "ansible-core${{ parameters.ansible_core_version }}"
+    retryCountOnTaskFailure: 5
+    displayName: Install tools
+
+  - script: |
+      rm -rf ~/.ansible
+      mkdir -p ~/.ansible
+      ln -snf $(readlink -f roles) ~/.ansible/roles
+      ln -snf $(readlink -f plugins) ~/.ansible/plugins
+    displayName: Setup ansible-freeipa using Git repository
+
+  - script: ansible-galaxy collection install containers.podman
+    displayName: Install Ansible Galaxy collections
+
+  - script: infra/image/build.sh -s ${{ parameters.distro }}
+    displayName: Build ${{ parameters.distro }} base image
+
+  - script: podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io
+    displayName: Registry login
+    env:
+      # Secrets needs to be mapped as env vars to work properly
+      QUAY_ROBOT_TOKEN: $(QUAY_ROBOT_TOKEN)
+
+  - script: |
+      podman push quay.io/ansible-freeipa/upstream-tests:${{parameters.distro}}-base quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-base
+    displayName: Push base image
+
+  - script: |
+      podman push quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server
+    displayName: Push server image

infra/azure/templates/fast_tests.yml

@@ -0,0 +1,31 @@
+---
+parameters:
+  - name: distro
+    type: string
+    default: fedora-latest
+  - name: build_number
+    type: string
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: target_python
+    type: string
+    default: "/usr/bin/python3"
+
+jobs:
+- template: playbook_fast.yml
+  parameters:
+    group_number: 1
+    number_of_groups: 1
+    build_number: ${{ parameters.build_number }}
+    distro: ${{ parameters.distro }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+    target_python: ${{ parameters.target_python }}
+
+# - template: pytest_tests.yml
+#   parameters:
+#     build_number: ${{ parameters.build_number }}
+#     distro: ${{ parameters.distro }}
+#     ansible_version: ${{ parameters.ansible_version }}
+#     python_version: '< 3.12'

infra/azure/templates/galaxy_pytest_script.yml

@@ -0,0 +1,65 @@
+---
+parameters:
+  - name: build_number
+    type: string
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+
+jobs:
+- job: Test_PyTests
+  displayName: Run pytests on ${{ parameters.scenario }}
+  timeoutInMinutes: 240
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install \
+        "molecule-plugins[docker]" \
+        "requests<2.29" \
+        "ansible${{ parameters.ansible_version }}"
+    retryCountOnTaskFailure: 5
+    displayName: Install molecule and Ansible
+
+  - script: ansible-galaxy collection install community.docker ansible.posix
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible collections
+
+  - script: pip install -r requirements-tests.txt
+    retryCountOnTaskFailure: 5
+    displayName: Install dependencies
+
+  - script: |
+      utils/build-galaxy-release.sh -i
+      molecule create -s ${{ parameters.scenario }}
+    retryCountOnTaskFailure: 5
+    displayName: Setup test container
+    env:
+      ANSIBLE_LIBRARY: ./molecule
+
+  - script: |
+      cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa
+      pytest \
+        -m "not playbook" \
+        --verbose \
+        --color=yes \
+        --junit-xml=TEST-results-pytests.xml
+    displayName: Run tests
+    env:
+      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      RUN_TESTS_IN_DOCKER: true
+      IPA_VERBOSITY: "-vvv"
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

infra/azure/templates/galaxy_script.yml

@@ -0,0 +1,86 @@
+---
+parameters:
+  - name: group_number
+    type: number
+    default: 1
+  - name: number_of_groups
+    type: number
+    default: 1
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+  - name: build_number
+    type: string
+
+
+jobs:
+- job: Test_Group${{ parameters.group_number }}
+  displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  timeoutInMinutes: 240
+  variables:
+  - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install \
+        "molecule-plugins[docker]" \
+        "requests<2.29" \
+        "ansible${{ parameters.ansible_version }}"
+    retryCountOnTaskFailure: 5
+    displayName: Install molecule and Ansible
+
+  - script: ansible-galaxy collection install community.docker ansible.posix
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible collections
+
+  - script: pip install -r requirements-tests.txt
+    retryCountOnTaskFailure: 5
+    displayName: Install dependencies
+
+  - script: |
+      utils/build-galaxy-release.sh -i
+      molecule create -s ${{ parameters.scenario }}
+    retryCountOnTaskFailure: 5
+    displayName: Setup test container
+    env:
+      ANSIBLE_LIBRARY: ./molecule
+
+  - script: |
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
+  - script: |
+      cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa
+      pytest \
+        -m "playbook" \
+        --verbose \
+        --color=yes \
+        --splits=${{ parameters.number_of_groups }} \
+        --group=${{ parameters.group_number }} \
+        --randomly-seed=$(date "+%Y%m%d") \
+        --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml
+    displayName: Run playbook tests
+    env:
+      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      RUN_TESTS_IN_DOCKER: true
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
+      IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
+      IPA_VERBOSITY: "-vvv"
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

infra/azure/templates/galaxy_tests.yml

@@ -0,0 +1,46 @@
+---
+parameters:
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: build_number
+    type: string
+  - name: ansible_version
+    type: string
+    default: ""
+
+jobs:
+- template: galaxy_script.yml
+  parameters:
+    group_number: 1
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+
+- template: galaxy_script.yml
+  parameters:
+    group_number: 2
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+
+- template: galaxy_script.yml
+  parameters:
+    group_number: 3
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+
+# Temporarily disable due to issues with ansible docker plugin.
+#- template: galaxy_pytest_script.yml
+#  parameters:
+#    build_number: ${{ parameters.build_number }}
+#    scenario: ${{ parameters.scenario }}
+#    ansible_version: ${{ parameters.ansible_version }}
+#    python_version: '< 3.12'

infra/azure/templates/group_tests.yml

@@ -0,0 +1,46 @@
+---
+parameters:
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: build_number
+    type: string
+  - name: ansible_version
+    type: string
+    default: ""
+
+jobs:
+- template: playbook_tests.yml
+  parameters:
+    group_number: 1
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+
+- template: playbook_tests.yml
+  parameters:
+    group_number: 2
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+
+- template: playbook_tests.yml
+  parameters:
+    group_number: 3
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+    python_version: '< 3.12'
+
+# Temporarily disabled due to ansible docker plugin issue.
+#- template: pytest_tests.yml
+#  parameters:
+#    build_number: ${{ parameters.build_number }}
+#    scenario: ${{ parameters.scenario }}
+#    ansible_version: ${{ parameters.ansible_version }}
+#    python_version: '< 3.12'

infra/azure/templates/playbook_fast.yml

@@ -0,0 +1,84 @@
+---
+parameters:
+  - name: group_number
+    type: number
+    default: 1
+  - name: number_of_groups
+    type: number
+    default: 1
+  - name: distro
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+  - name: build_number
+    type: string
+  - name: target_python
+    type: string
+    default: "/usr/bin/python3"
+
+jobs:
+- job: Test_Group${{ parameters.group_number }}
+  displayName: Run playbook tests ${{ parameters.distro }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  timeoutInMinutes: 360
+  variables:
+  - template: variables.yaml
+  - template: variables_${{ parameters.distro }}.yaml
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install "ansible${{ parameters.ansible_version }}"
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible
+
+  - script: ansible-galaxy collection install containers.podman
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible collections
+
+  - script: pip install -r requirements-tests.txt
+    retryCountOnTaskFailure: 5
+    displayName: Install dependencies
+
+  - script: |
+      . utils/set_test_modules
+      python3 utils/check_test_configuration.py ${{ parameters.distro }}
+    displayName: Check test configuration
+
+  - script: |
+      utils/setup_test_container.sh \
+        -e podman \
+        -a \
+        -m 4 \
+        -n "ipaserver.test.local" \
+        -p ${{ parameters.target_python }} \
+        -i ${{ parameters.distro }}-server \
+        ${{ parameters.distro }}-test
+    displayName: Setup target container
+
+  - script: |
+      . utils/set_test_modules
+      pytest -m "playbook" --verbose --color=yes --suppress-no-test-exit-code --junit-xml=TEST-results-pr-check.xml
+    displayName: Run playbook tests
+    env:
+      ANSIBLE_ROLES_PATH: "${PWD}/roles"
+      ANSIBLE_LIBRARY: "${PWD}/plugins"
+      ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils"
+      IPA_SERVER_HOST: ${{ parameters.distro }}-test
+      RUN_TESTS_IN_DOCKER: podman
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_VERBOSITY: "-vvv"
+      IPA_PYTHON_PATH: ${{ parameters.target_python }}
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

infra/azure/templates/playbook_tests.yml

@@ -0,0 +1,88 @@
+---
+parameters:
+  - name: group_number
+    type: number
+    default: 1
+  - name: number_of_groups
+    type: number
+    default: 1
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+  - name: build_number
+    type: string
+
+jobs:
+- job: Test_Group${{ parameters.group_number }}
+  displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  timeoutInMinutes: 240
+  variables:
+  - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install \
+        "molecule-plugins[docker]" \
+        "requests<2.29" \
+        "ansible${{ parameters.ansible_version }}"
+    retryCountOnTaskFailure: 5
+    displayName: Install molecule and Ansible
+
+  - script: ansible-galaxy collection install community.docker ansible.posix
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible collections
+
+  - script: pip install -r requirements-tests.txt
+    retryCountOnTaskFailure: 5
+    displayName: Install dependencies
+
+  - script: |
+      rm -rf ~/ansible
+      mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils
+      cp -a roles/* ~/.ansible/roles
+      cp -a plugins/modules/* ~/.ansible/library
+      cp -a plugins/module_utils/* ~/.ansible/module_utils
+      molecule create -s ${{ parameters.scenario }}
+    retryCountOnTaskFailure: 5
+    displayName: Setup test container
+    env:
+      ANSIBLE_LIBRARY: ./molecule
+
+  - script: |
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
+  - script: |
+      pytest \
+        -m "playbook" \
+        --verbose \
+        --color=yes \
+        --splits=${{ parameters.number_of_groups }} \
+        --group=${{ parameters.group_number }} \
+        --randomly-seed=$(date "+%Y%m%d") \
+        --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml
+    displayName: Run playbook tests
+    env:
+      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      RUN_TESTS_IN_DOCKER: true
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
+      IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
+      IPA_VERBOSITY: "-vvv"
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

infra/azure/templates/pytest_tests.yml

@@ -0,0 +1,75 @@
+---
+parameters:
+  - name: build_number
+    type: string
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+
+jobs:
+- job: Test_PyTests
+  displayName: Run pytests on ${{ parameters.scenario }}
+  timeoutInMinutes: 240
+  variables:
+  - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install \
+        "molecule-plugins[docker]" \
+        "requests<2.29" \
+        "ansible${{ parameters.ansible_version }}"
+    retryCountOnTaskFailure: 5
+    displayName: Install molecule and Ansible
+
+  - script: ansible-galaxy collection install community.docker ansible.posix
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible collections
+
+  - script: pip install -r requirements-tests.txt
+    retryCountOnTaskFailure: 5
+    displayName: Install dependencies
+
+  - script: |
+      rm -rf ~/.ansible
+      mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils
+      cp -a roles/* ~/.ansible/roles
+      cp -a plugins/modules/* ~/.ansible/library
+      cp -a plugins/module_utils/* ~/.ansible/module_utils
+      molecule create -s ${{ parameters.scenario }}
+    retryCountOnTaskFailure: 5
+    displayName: Setup test container
+    env:
+      ANSIBLE_LIBRARY: ./molecule
+
+  - script: |
+      pytest \
+        -m "not playbook" \
+        --verbose \
+        --color=yes \
+        --junit-xml=TEST-results-pytests.xml
+    displayName: Run tests
+    env:
+      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      RUN_TESTS_IN_DOCKER: true
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
+      IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
+      IPA_VERBOSITY: "-vvv"
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

infra/azure/templates/variables.yaml

@@ -0,0 +1,23 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# If no variable is to be set, add 'empty: true', as the
+# 'variables' dict cannot be empty.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  # empty: true
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  ipa_disabled_modules: >-
+    config
+  # ipa_disabled_tests: >-

infra/azure/templates/variables_c8s.yaml

@@ -0,0 +1,21 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# ipa_disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+# If no variables are set, set "empty: true" as at least
+# one item is needed in the set.
+---
+variables:
+  empty: true
+#   ipa_enabled_modules: >-
+#   ipa_enabled_tests: >-
+#   ipa_disabled_modules: >-
+#   ipa_disabled_tests: >-

infra/azure/templates/variables_c9s.yaml

@@ -0,0 +1,21 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# ipa_disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+# If no variables are set, set "empty: true" as at least
+# one item is needed in the set.
+---
+variables:
+  empty: true
+#   ipa_enabled_modules: >-
+#   ipa_enabled_tests: >-
+#   ipa_disabled_modules: >-
+#   ipa_disabled_tests: >-

infra/azure/templates/variables_centos-7.yaml

@@ -0,0 +1,20 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# ipa_disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+# If no variables are set, set "empty: true" as at least
+# one item is needed in the set.
+---
+variables:
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  # ipa_disabled_modules: >-
+  ipa_disabled_tests: test_config_sid

infra/azure/templates/variables_fedora-latest.yaml

@@ -0,0 +1,22 @@
+#
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# ipa_disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+# If no variables are set, set "empty: true" as at least
+# one item is needed in the set.
+---
+variables:
+  empty: true
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  # ipa_disabled_modules: >-
+  # ipa_disabled_tests: >-

infra/azure/templates/variables_fedora-rawhide.yaml

@@ -0,0 +1,21 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# ipa_disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+# If no variables are set, set "empty: true" as at least
+# one item is needed in the set.
+---
+variables:
+  empty: true
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  # ipa_disabled_modules: >-
+  # ipa_disabled_tests: >-