YUM4/DNF compatibility via yum action plugin (#44322)

* YUM4/DNF compatibility via yum action plugin

DNF does not natively support allow_downgrade as an option, instead
that is always the default (not configurable by the administrator)
so it had to be implemented

 - Fixed group actions in check mode to report correct changed state
 - Better error handling for depsolve and transaction errors in DNF
 - Fixed group action idempotent transactions
 - Add use_backend to yum module/action plugin
 - Fix dnf handling of autoremove (didn't used to work nor had a
   default value specified, now does work and matches default
   behavior of yum)
 - Enable installroot tests for yum4(dnf) integration testing, dnf
   backend now supports that
 - Switch from zip to bc for certain package install/remove test
   cases in yum integration tests. The dnf depsolver downgrades
   python when you uninstall zip which alters the test environment
   and we have no control over that.
 - Add changelog fragment
 - Return a pkg_mgr fact if it was not previously set.

test/integration/targets/dnf/tasks/dnf.yml

@@ -301,6 +301,11 @@
         - "'changed' in dnf_result"
         - "'msg' in dnf_result"
 
+- name: verify that bc is not installed
+  dnf:
+    name: bc
+    state: absent
+
 - name: install the group again but also with a package that is not yet installed
   dnf:
     name:

test/integration/targets/dnf/tasks/repo.yml

@@ -87,6 +87,7 @@
       dnf:
         name: "{{ repodir }}/foo-1.0-1.{{ ansible_architecture }}.rpm"
         state: present
+        allow_downgrade: True
       register: dnf_result
 
     - name: Check foo with rpm

test/integration/targets/setup_rpm_repo/tasks/main.yml

@@ -42,7 +42,7 @@
       mode: 0755
 
   - name: Create RPMs and put them into a repo
-    shell: "python /tmp/create-repo.py {{ ansible_architecture }}"
+    shell: "{{ansible_python_interpreter}} /tmp/create-repo.py {{ ansible_architecture }}"
     register: repo
 
   - set_fact:
@@ -56,7 +56,7 @@
       gpgcheck: no
 
   - name: Create RPMs and put them into a repo (i686)
-    shell: "python /tmp/create-repo.py i686"
+    shell: "{{ansible_python_interpreter}} /tmp/create-repo.py i686"
     register: repo_i686
 
   - set_fact:
@@ -70,7 +70,7 @@
       gpgcheck: no
 
   - name: Create RPMs and put them into a repo (ppc64)
-    shell: "python /tmp/create-repo.py ppc64"
+    shell: "{{ansible_python_interpreter}} /tmp/create-repo.py ppc64"
     register: repo_ppc64
 
   - set_fact:

test/integration/targets/yum/tasks/main.yml

@@ -46,8 +46,8 @@
         - ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and ansible_distribution_major_version|int <= 6
   when:
     - ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux', 'Fedora']
-    - ansible_python.version.major == 2
 
+# DNF1 doesn't handle downgrade operations properly (Fedora < 26)
 - block:
     - include: 'repo.yml'
   always:
@@ -58,16 +58,10 @@
     - command: yum clean metadata
   when:
     - ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux', 'Fedora']
-    - ansible_python.version.major == 2
 
-# We can't run yum --installroot tests on dnf systems.  Dnf systems revert to
-# yum-deprecated, and yum-deprecated refuses to run if yum.conf exists
-# so we cannot configure yum-deprecated correctly in an empty /tmp/fake.root/
-# It will always run with $releasever unset
 - include: 'yuminstallroot.yml'
   when:
-    - (ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] or (ansible_distribution in ['Fedora'] and ansible_distribution_major_version|int < 23))
-    - ansible_python.version.major == 2
+    - ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux', 'Fedora']
 
 # el6 has a broken yum group implementation, when you try to remove a group it goes through
 # deps and ends up with trying to remove yum itself and the whole process fails
@@ -75,4 +69,3 @@
 - include: 'yum_group_remove.yml'
   when:
     - (ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and ansible_distribution_major_version|int > 6) or ansible_distribution in ['Fedora']
-    - ansible_python.version.major == 2

test/integration/targets/yum/tasks/repo.yml

@@ -73,10 +73,11 @@
         name: foo
         state: absent
     # ============================================================================
-    - name: Install 1:foo-1.0-2
+    - name: Downgrade foo
       yum:
-        name: "1:foo-1.0-2.{{ ansible_architecture }}"
+        name: foo-1.0-1
         state: present
+        allow_downgrade: yes
       register: yum_result
 
     - name: Check foo with rpm
@@ -87,30 +88,7 @@
       assert:
         that:
             - "yum_result.changed"
-            - "rpm_result.stdout.startswith('foo-1.0-2')"
-
-    - name: Verify yum module outputs
-      assert:
-        that:
-            - "'msg' in yum_result"
-            - "'rc' in yum_result"
-            - "'results' in yum_result"
-    # ============================================================================
-    - name: Install foo-1.0-2 again
-      yum:
-        name: foo-1.0-2
-        state: present
-      register: yum_result
-
-    - name: Check foo with rpm
-      shell: rpm -q foo
-      register: rpm_result
-
-    - name: Verify installation
-      assert:
-        that:
-            - "not yum_result.changed"
-            - "rpm_result.stdout.startswith('foo-1.0-2')"
+            - "rpm_result.stdout.startswith('foo-1.0-1')"
 
     - name: Verify yum module outputs
       assert:
@@ -279,30 +257,6 @@
             - "not yum_result.changed"
             - "rpm_result.stdout.startswith('foo-1.0-2')"
 
-    - name: Verify yum module outputs
-      assert:
-        that:
-            - "'msg' in yum_result"
-            - "'rc' in yum_result"
-            - "'results' in yum_result"
-    # ============================================================================
-    - name: Downgrade foo
-      yum:
-        name: foo-1.0-1
-        state: present
-        allow_downgrade: yes
-      register: yum_result
-
-    - name: Check foo with rpm
-      shell: rpm -q foo
-      register: rpm_result
-
-    - name: Verify installation
-      assert:
-        that:
-            - "yum_result.changed"
-            - "rpm_result.stdout.startswith('foo-1.0-1')"
-
     - name: Verify yum module outputs
       assert:
         that:
@@ -506,3 +460,99 @@
       yum:
         name: foo
         state: absent
+
+# FIXME: dnf currently doesn't support epoch as part of it's pkg_spec for
+#        finding install candidates
+#         https://bugzilla.redhat.com/show_bug.cgi?id=1619687
+- block:
+    - name: Install 1:foo-1.0-2
+      yum:
+        name: "1:foo-1.0-2.{{ ansible_architecture }}"
+        state: present
+      register: yum_result
+
+    - name: Check foo with rpm
+      shell: rpm -q foo
+      register: rpm_result
+
+    - name: Verify installation
+      assert:
+        that:
+            - "yum_result.changed"
+            - "rpm_result.stdout.startswith('foo-1.0-2')"
+
+    - name: Verify yum module outputs
+      assert:
+        that:
+            - "'msg' in yum_result"
+            - "'rc' in yum_result"
+            - "'results' in yum_result"
+  always:
+    - name: Clean up
+      yum:
+        name: foo
+        state: absent
+
+  when: ansible_pkg_mgr == 'yum'
+
+# DNF1 (Fedora < 26) had some issues:
+# - did not accept architecture tag as valid component of a package spec unless
+#   installing a file (i.e. can't search the repo)
+# - doesn't handle downgrade transactions via the API properly, marks it as a
+#   conflict
+#
+# NOTE: Both DNF1 and Fedora < 26 have long been EOL'd by their respective
+#       upstreams
+- block:
+    # ============================================================================
+    - name: Install foo-1.0-2
+      yum:
+        name: "foo-1.0-2.{{ ansible_architecture }}"
+        state: present
+      register: yum_result
+
+    - name: Check foo with rpm
+      shell: rpm -q foo
+      register: rpm_result
+
+    - name: Verify installation
+      assert:
+        that:
+            - "yum_result.changed"
+            - "rpm_result.stdout.startswith('foo-1.0-2')"
+
+    - name: Verify yum module outputs
+      assert:
+        that:
+            - "'msg' in yum_result"
+            - "'rc' in yum_result"
+            - "'results' in yum_result"
+
+    - name: Install foo-1.0-2 again
+      yum:
+        name: foo-1.0-2
+        state: present
+      register: yum_result
+
+    - name: Check foo with rpm
+      shell: rpm -q foo
+      register: rpm_result
+
+    - name: Verify installation
+      assert:
+        that:
+            - "not yum_result.changed"
+            - "rpm_result.stdout.startswith('foo-1.0-2')"
+
+    - name: Verify yum module outputs
+      assert:
+        that:
+            - "'msg' in yum_result"
+            - "'rc' in yum_result"
+            - "'results' in yum_result"
+  always:
+    - name: Clean up
+      yum:
+        name: foo
+        state: absent
+  when: not (ansible_distribution == "Fedora" and ansible_distribution_major_version|int < 26)

test/integration/targets/yum/tasks/yum.yml

@@ -477,6 +477,11 @@
 
 - set_fact:
     pkg_url: https://s3.amazonaws.com/ansible-ci-files/test/integration/targets/yum/fpaste-0.3.7.4.1-2.el7.noarch.rpm
+  when: ansible_python.version.major == 2
+
+- set_fact:
+    pkg_url: https://s3.amazonaws.com/ansible-ci-files/test/integration/targets/yum/fpaste-0.3.9.2-1.fc28.noarch.rpm
+  when: ansible_python.version.major == 3
 # setup end
 
 - name: download an rpm
@@ -566,7 +571,6 @@
     that:
       - "'changed' in no_nevra_info_result"
       - "'msg' in no_nevra_info_result"
-      - "'Failed to get nevra information from RPM package' in no_nevra_info_result.msg"
 
 - name: Delete a temp RPM file
   file:
@@ -583,102 +587,131 @@
     yum_version: "{%- if item.yumstate == 'installed' -%}{{ item.version }}{%- else -%}{{ yum_version }}{%- endif -%}"
   with_items: "{{ yum_version.results }}"
 
-- name: check whether yum supports disableexcludes (>= 3.4)
-  set_fact:
-    supports_disable_excludes: "{{ yum_version is version_compare('3.4.0', '>=') }}"
+- block:
+  - name: check whether yum supports disableexcludes (>= 3.4)
+    set_fact:
+      supports_disable_excludes: "{{ yum_version is version_compare('3.4.0', '>=') }}"
+    when: ansible_pkg_mgr == "yum"
 
-- name: uninstall zip
-  yum: name=zip state=removed
+  - name: unset disableexcludes tests for dnf(yum4) backend temporarily
+    set_fact:
+      supports_disable_excludes: True
+    when: ansible_pkg_mgr == "dnf"
 
-- name: check zip with rpm
-  shell: rpm -q zip
-  ignore_errors: True
-  register: rpm_zip_result
+  - name: uninstall bc
+    yum: name=bc state=removed
 
-- name: verify zip is uninstalled
-  assert:
-    that:
-      - "rpm_zip_result is failed"
+  - name: check bc with rpm
+    shell: rpm -q bc
+    ignore_errors: True
+    register: rpm_bc_result
 
-- name: exclude zip
-  lineinfile:
-    dest: /etc/yum.conf
-    regexp: (^exclude=)(.)*
-    line: "exclude=zip*"
-    state: present
+  - name: verify bc is uninstalled
+    assert:
+      that:
+        - "rpm_bc_result is failed"
 
-# begin test case where disable_excludes is supported
-- name: Try install zip without disable_excludes
-  yum: name=zip state=latest
-  register: yum_zip_result
-  ignore_errors: True
-  when: supports_disable_excludes
+  - name: exclude bc (yum backend)
+    lineinfile:
+      dest: /etc/yum.conf
+      regexp: (^exclude=)(.)*
+      line: "exclude=bc*"
+      state: present
+    when: ansible_pkg_mgr == 'yum'
 
-- name: verify zip did not install because it is in exclude list
-  assert:
-    that:
-      - "yum_zip_result is failed"
-  when: supports_disable_excludes
+  - name: exclude bc (dnf backend)
+    lineinfile:
+      dest: /etc/dnf/dnf.conf
+      regexp: (^excludepkgs=)(.)*
+      line: "excludepkgs=bc*"
+      state: present
+    when: ansible_pkg_mgr == 'dnf'
 
-- name: install zip with disable_excludes
-  yum: name=zip state=latest disable_excludes=all
-  register: yum_zip_result_using_excludes
-  when: supports_disable_excludes
+  # begin test case where disable_excludes is supported
+  - name: Try install bc without disable_excludes
+    yum: name=bc state=latest
+    register: yum_bc_result
+    ignore_errors: True
+    when: supports_disable_excludes
 
-- name: verify zip did install using disable_excludes=all
-  assert:
-    that:
-      - "yum_zip_result_using_excludes is success"
-      - "yum_zip_result_using_excludes is changed"
-      - "yum_zip_result_using_excludes is not failed"
-  when: supports_disable_excludes
+  - name: verify bc did not install because it is in exclude list
+    assert:
+      that:
+        - "yum_bc_result is failed"
+    when: supports_disable_excludes
 
-- name: remove exclude zip (cleanup yum.conf)
-  lineinfile:
-    dest: /etc/yum.conf
-    regexp: (^exclude=zip*)
-    line: "exclude="
-    state: present
-  when: supports_disable_excludes
-# end test case where disable_excludes is supported
+  - name: install bc with disable_excludes
+    yum: name=bc state=latest disable_excludes=all
+    register: yum_bc_result_using_excludes
+    when: supports_disable_excludes
 
-# begin test case where disable_excludes is not supported
-- name: Try install zip with disable_excludes
-  yum: name=zip state=latest disable_excludes=all
-  register: yum_fail_zip_result_old_yum
-  ignore_errors: True
-  when: not supports_disable_excludes
+  - name: verify bc did install using disable_excludes=all
+    assert:
+      that:
+        - "yum_bc_result_using_excludes is success"
+        - "yum_bc_result_using_excludes is changed"
+        - "yum_bc_result_using_excludes is not failed"
+    when: supports_disable_excludes
 
-- name: verify packages did not install because yum version is unsupported
-  assert:
-    that:
-      - "yum_fail_zip_result_old_yum is failed"
-  when: not supports_disable_excludes
+  - name: remove exclude bc (cleanup yum.conf)
+    lineinfile:
+      dest: /etc/yum.conf
+      regexp: (^exclude=bc*)
+      line: "exclude="
+      state: present
+    when: supports_disable_excludes and (ansible_pkg_mgr == 'yum')
 
-- name: verify yum module outputs
-  assert:
-    that:
-        - "'is available in yum version 3.4 and onwards.' in yum_fail_zip_result_old_yum.msg"
-  when: not supports_disable_excludes
+  - name: remove exclude bc (cleanup dnf.conf)
+    lineinfile:
+      dest: /etc/dnf/dnf.conf
+      regexp: (^excludepkgs=bc*)
+      line: "excludepkgs="
+      state: present
+    when: ansible_pkg_mgr == 'dnf'
+  # end test case where disable_excludes is supported
 
-- name: remove exclude zip (cleanup yum.conf)
-  lineinfile:
-    dest: /etc/yum.conf
-    regexp: (^exclude=zip*)
-    line: "exclude="
-    state: present
-  when: not supports_disable_excludes
+  # begin test case where disable_excludes is not supported
+  - name: Try install bc with disable_excludes
+    yum: name=bc state=latest disable_excludes=all
+    register: yum_fail_bc_result_old_yum
+    ignore_errors: True
+    when: not supports_disable_excludes
 
-- name: install zip (bring test env in same state as when testing started)
-  yum: name=zip state=latest
-  register: yum_zip_result_old_yum
-  when: not supports_disable_excludes
+  - name: verify packages did not install because yum version is unsupported
+    assert:
+      that:
+        - "yum_fail_bc_result_old_yum is failed"
+    when: not supports_disable_excludes
 
-- name: verify zip installed
-  assert:
-    that:
-      - "yum_zip_result_old_yum is success"
-      - "yum_zip_result_old_yum is changed"
-      - "yum_zip_result_old_yum is not failed"
-  when: not supports_disable_excludes
-# end test case where disable_excludes is not supported
+  - name: verify yum module outputs
+    assert:
+      that:
+          - "'is available in yum version 3.4 and onwards.' in yum_fail_bc_result_old_yum.msg"
+    when: not supports_disable_excludes
+
+  - name: remove exclude bc (cleanup yum.conf)
+    lineinfile:
+      dest: /etc/yum.conf
+      regexp: (^exclude=bc*)
+      line: "exclude="
+      state: present
+    when: not supports_disable_excludes and ansible_pkg_mgr == 'yum'
+
+  - name: install bc (bring test env in same state as when testing started)
+    yum: name=bc state=latest
+    register: yum_bc_result_old_yum
+    when: not supports_disable_excludes
+
+  - name: verify bc installed
+    assert:
+      that:
+        - "yum_bc_result_old_yum is success"
+        - "yum_bc_result_old_yum is changed"
+        - "yum_bc_result_old_yum is not failed"
+    when: not supports_disable_excludes and ansible_pkg_mgr == "yum"
+  # end test case where disable_excludes is not supported
+
+  # Fedora < 26 has a bug in dnf where package excludes in dnf.conf aren't
+  # actually honored and those releases are EOL'd so we have no expectation they
+  # will ever be fixed
+  when: not ((ansible_distribution == "Fedora") and (ansible_distribution_major_version|int < 26))

test/integration/targets/yum/tasks/yum_group_remove.yml

@@ -5,6 +5,16 @@
   with_items:
     - "@Development Tools"
     - yum-utils
+  when: ansible_pkg_mgr == "yum"
+
+- name: install a group to test and dnf-utils
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "@Development Tools"
+    - dnf-utils
+  when: ansible_pkg_mgr == "dnf"
 
 - name: check mode remove the group
   yum:

test/integration/targets/yum_repository/tasks/yum_repository_fedora.yml

@@ -36,7 +36,7 @@
   assert:
     that:
       - lib_result.failed
-      - "lib_result.msg=='No package libbdplus available.'"
+      - "lib_result.msg=='Failed to install some of the specified packages'"
 
 - name: re-add rpmfusion
   yum_repository: