Merge pull request #1225 from rjeffman/ci_pin_ansible_lint_version

Bump linter tools versions an fix linter errors

.github/workflows/ansible-test.yml

@@ -8,7 +8,7 @@ jobs:
     name: Verify ansible-test sanity
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
       - name: Run ansible-test

.github/workflows/docs.yml

@@ -8,10 +8,10 @@ jobs:
     name: Check Ansible Documentation with ansible-core 2.13.
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.x'
       - name: Install Ansible 2.13
@@ -25,10 +25,10 @@ jobs:
     name: Check Ansible Documentation with ansible-core 2.14.
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.x'
       - name: Install Ansible 2.14
@@ -42,10 +42,10 @@ jobs:
     name: Check Ansible Documentation with ansible-core 2.15.
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.x'
       - name: Install Ansible 2.15
@@ -59,10 +59,10 @@ jobs:
     name: Check Ansible Documentation with latest Ansible version.
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.x'
       - name: Install Ansible-latest

.github/workflows/lint.yml

@@ -8,15 +8,15 @@ jobs:
     name: Verify ansible-lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.x"
       - name: Run ansible-lint
         run: |
-          pip install "ansible-core>=2.16,<2.17" 'ansible-lint>=6.22'
+          pip install "ansible-core>=2.16,<2.17" 'ansible-lint==6.22'
           utils/build-galaxy-release.sh -ki
           cd .galaxy-build
           ansible-lint --profile production --exclude tests/integration/ --exclude tests/unit/ --parseable --nocolor
@@ -25,10 +25,10 @@ jobs:
     name: Verify yamllint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.x"
       - name: Run yaml-lint
@@ -38,10 +38,10 @@ jobs:
     name: Verify pydocstyle
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.x"
       - name: Run pydocstyle
@@ -53,10 +53,10 @@ jobs:
     name: Verify flake8
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.x"
       - name: Run flake8
@@ -68,10 +68,10 @@ jobs:
     name: Verify pylint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4.3.0
+          fetch-depth: 1
+      - uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.x"
       - name: Run pylint
@@ -83,8 +83,8 @@ jobs:
     name: Shellcheck
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master

.github/workflows/readme.yml

@@ -8,9 +8,9 @@ jobs:
     name: Verify readme
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v4.1.1
         with:
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Run readme test
         run: |
           error=0

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
 - repo: https://github.com/ansible/ansible-lint.git
-  rev: v6.22.0
+  rev: v24.5.0
   hooks:
   - id: ansible-lint
     always_run: false
@@ -21,20 +21,20 @@ repos:
           --parseable
           --nocolor
 - repo: https://github.com/adrienverge/yamllint.git
-  rev: v1.32.0
+  rev: v1.35.1
   hooks:
   - id: yamllint
     files: \.(yaml|yml)$
 - repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
+  rev: 7.0.0
   hooks:
   - id: flake8
 - repo: https://github.com/pycqa/pydocstyle
-  rev: 6.0.0
+  rev: 6.3.0
   hooks:
   - id: pydocstyle
 - repo: https://github.com/pycqa/pylint
-  rev: v3.0.2
+  rev: v3.2.2
   hooks:
   - id: pylint
     args:

README-dnszone.md

@@ -135,7 +135,7 @@ Example playbook to enable a zone:
 
 Example playbook to allow per-zone privilege delegation:
 
-``` yaml
+```yaml
 ---
 - name: Playbook to enable per-zone privilege delegation
   hosts: ipaserver

README-group.md

@@ -158,7 +158,7 @@ Several groups can also be renamed with a single task, as in the example playboo
   gather_facts: false
 
   tasks:
-  - name Rename group1 to newgroup1 and group2 to newgroup2
+  - name: Rename group1 to newgroup1 and group2 to newgroup2
     ipagroup:
       ipaadmin_password: SomeADMINpassword
       groups:

README-inventory-plugin-freeipa.md

@@ -0,0 +1,106 @@
+Inventory plugin
+================
+
+Description
+-----------
+
+
+The inventory plugin compiles a dynamic inventory from IPA domain. The servers can be filtered by their role(s).
+
+This plugin is using the Python requests binding, that is only available for Python 3.7 and up.
+
+
+Features
+--------
+* Dynamic inventory
+
+
+Supported FreeIPA Versions
+--------------------------
+
+FreeIPA versions 4.6.0 and up are supported by the inventory plugin.
+
+
+Requirements
+------------
+
+**Controller**
+* Ansible version: 2.13+
+
+**Node**
+* Supported FreeIPA version (see above)
+
+
+Configuration
+=============
+
+The inventory plugin is automatically enabled from the Ansible collection or from the top directory of the git repo if the `plugins` folder is linked to `~/.ansible`.
+
+If `ansible.cfg` was modified to point to the roles and modules with `roles_path`, `library` and `module_utils` tag, then it is needed to set `inventory_plugins` also:
+
+```
+inventory_plugins = /my/dir/ansible-freeipa/plugins/inventory
+```
+
+Usage
+=====
+
+Example inventory file "freeipa.yml":
+
+```yml
+---
+plugin: freeipa
+server: server.ipa.local
+ipaadmin_password: SomeADMINpassword
+```
+
+Example inventory file "freeipa.yml" with server TLS certificate verification using local copy of `/etc/ipa/ca.crt` from the server:
+
+```yml
+---
+plugin: freeipa
+server: server.ipa.local
+ipaadmin_password: SomeADMINpassword
+verify: ca.crt
+```
+
+
+How to use the plugin
+---------------------
+
+With the `ansible-inventory` command it is possible to show the generated inventorey:
+
+```bash
+ansible-inventory -v -i freeipa.yml --graph
+```
+
+Example inventory file "freeipa.yml" for use with `playbooks/config/retrieve-config.yml`:
+
+```yml
+---
+plugin: freeipa
+server: server.ipa.local
+ipaadmin_password: SomeADMINpassword
+inventory_group: ipaserver
+```
+
+```bash
+ansible-playbook -u root -i ipa.yml playbooks/config/retrieve-config.yml 
+```
+
+Variables
+=========
+
+Variable | Description | Required
+-------- | ----------- | --------
+`ipaadmin_principal` | The admin principal is a string and defaults to `admin` | no
+`ipaadmin_password` | The admin password is a string and is required if there is no admin ticket available on the node | no
+`server` | The FQDN of server to start the scan. (string) | yes
+`verify` | The server TLS certificate file for verification (/etc/ipa/ca.crt). Turned off if not set. (string) | yes
+`role` | The role(s) of the server. If several roles are given, only servers that have all the roles are returned. (list of strings) (choices: "IPA master", "CA server", "KRA server", "DNS server", "AD trust controller", "AD trust agent") | no
+`inventory_group` | The inventory group to create. The default group name is "ipaservers". | no
+
+Authors
+=======
+
+- Thomas Woerner

README-user.md

@@ -311,7 +311,7 @@ Example playbook to rename users:
       ipaadmin_password: SomeADMINpassword
       name: pinky
       rename: reddy
-      state: enabled
+      state: renamed
 ```
 
 Example playbook to unlock users:

README.md

@@ -13,6 +13,7 @@ Features
 * Repair mode for clients
 * Backup and restore, also to and from controller
 * Smartcard setup for servers and clients
+* Inventory plugin freeipa
 * Modules for automembership rule management
 * Modules for automount key management
 * Modules for automount location management
@@ -108,9 +109,10 @@ You can use the roles directly within the top directory of the git repo, but to
 You can either adapt ansible.cfg:
 
 ```
-roles_path   = /my/dir/ansible-freeipa/roles
-library      = /my/dir/ansible-freeipa/plugins/modules
-module_utils = /my/dir/ansible-freeipa/plugins/module_utils
+roles_path        = /my/dir/ansible-freeipa/roles
+library           = /my/dir/ansible-freeipa/plugins/modules
+module_utils      = /my/dir/ansible-freeipa/plugins/module_utils
+inventory_plugins = /my/dir/ansible-freeipa/plugins/inventory
 ```
 
 Or you can link the directories:
@@ -470,3 +472,8 @@ Modules in plugin/modules
 * [ipavault](README-vault.md)
 
 If you want to write a new module please read [writing a new module](plugins/modules/README.md).
+
+Inventory plugins in plugin/inventory
+=====================================
+
+* [freeipa](README-inventory-plugin-freeipa.md)

plugins/inventory/freeipa.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+
+# Authors:
+#   Thomas Woerner <twoerner@redhat.com>
+#
+# Copyright (C) 2024 Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+ANSIBLE_METADATA = {
+    "metadata_version": "1.0",
+    "supported_by": "community",
+    "status": ["preview"],
+}
+
+DOCUMENTATION = """
+---
+name: freeipa
+plugin_type: inventory
+version_added: "1.13"
+short_description: Compiles a dynamic inventory from IPA domain
+description: |
+  Compiles a dynamic inventory from IPA domain, filters servers by role(s).
+options:
+  plugin:
+    description: Marks this as an instance of the "freeipa" plugin.
+    required: True
+    choices: ["freeipa"]
+  ipaadmin_principal:
+    description: The admin principal.
+    default: admin
+    type: str
+  ipaadmin_password:
+    description: The admin password.
+    required: true
+    type: str
+  server:
+    description: FQDN of server to start the scan.
+    type: str
+    required: true
+  verify:
+    description: |
+      The server TLS certificate file for verification (/etc/ipa/ca.crt).
+      Turned off if not set.
+    type: str
+    required: false
+  role:
+    description: |
+      The role(s) of the server. If several roles are given, only servers
+      that have all the roles are returned.
+    type: list
+    elements: str
+    choices: ["IPA master", "CA server", "KRA server", "DNS server",
+              "AD trust controller", "AD trust agent"]
+    required: false
+  inventory_group:
+    description: |
+      The inventory group to create. The default group name is "ipaservers".
+    type: str
+    default: ipaservers
+author:
+  - Thomas Woerner (@t-woerner)
+"""
+
+EXAMPLES = """
+# inventory.config file in YAML format
+plugin: freeipa
+server: ipaserver-01.ipa.local
+ipaadmin_password: SomeADMINpassword
+
+# inventory.config file in YAML format with server TLS certificate verification
+plugin: freeipa
+server: ipaserver-01.ipa.local
+ipaadmin_password: SomeADMINpassword
+verify: ca.crt
+"""
+
+import os
+import requests
+try:
+    from requests.packages import urllib3
+except ImportError:
+    import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+from ansible import constants
+from ansible.errors import AnsibleParserError
+from ansible.module_utils.common.text.converters import to_native
+from ansible.plugins.inventory import BaseInventoryPlugin
+from ansible.module_utils.six.moves.urllib.parse import quote
+
+
+class InventoryModule(BaseInventoryPlugin):
+
+    NAME = 'freeipa'
+
+    def verify_file(self, path):
+        # pylint: disable=super-with-arguments
+        if super(InventoryModule, self).verify_file(path):
+            _name, ext = os.path.splitext(path)
+            if ext in constants.YAML_FILENAME_EXTENSIONS:
+                return True
+        return False
+
+    def parse(self, inventory, loader, path, cache=False):
+        # pylint: disable=super-with-arguments
+        super(InventoryModule, self).parse(inventory, loader, path,
+                                           cache=cache)
+        self._read_config_data(path)  # This also loads the cache
+
+        self.get_option("plugin")
+        ipaadmin_principal = self.get_option("ipaadmin_principal")
+        ipaadmin_password = self.get_option("ipaadmin_password")
+        server = self.get_option("server")
+        verify = self.get_option("verify")
+        role = self.get_option("role")
+        inventory_group = self.get_option("inventory_group")
+
+        if verify is not None:
+            if not os.path.exists(verify):
+                raise AnsibleParserError("ERROR: Could not load %s" % verify)
+        else:
+            verify = False
+
+        self.inventory.add_group(inventory_group)
+
+        ipa_url = "https://%s/ipa" % server
+
+        s = requests.Session()
+        s.headers.update({"referer": ipa_url})
+        s.headers.update({"Content-Type":
+                          "application/x-www-form-urlencoded"})
+        s.headers.update({"Accept": "text/plain"})
+
+        data = 'user=%s&password=%s' % (quote(ipaadmin_principal, safe=''),
+                                        quote(ipaadmin_password, safe=''))
+        response = s.post("%s/session/login_password" % ipa_url,
+                          data=data, verify=verify)
+
+        # Now use json API
+        s.headers.update({"Content-Type": "application/json"})
+
+        kw_args = {}
+        if role is not None:
+            kw_args["servrole"] = role
+        json_data = {
+            "method" : "server_find",
+            "params": [[], kw_args],
+            "id": 0
+        }
+        response = s.post("%s/session/json" % ipa_url, json=json_data,
+                          verify=verify)
+        json_res = response.json()
+
+        error = json_res.get("error")
+        if error is not None:
+            raise AnsibleParserError("ERROR: %s" % to_native(error))
+
+        if "result" in json_res and "result" in json_res["result"]:
+            res = json_res["result"].get("result")
+            if isinstance(res, list):
+                for server in res:
+                    self.inventory.add_host(server["cn"][0],
+                                            group=inventory_group)

plugins/module_utils/ansible_freeipa_module.py

@@ -25,13 +25,24 @@ from __future__ import (absolute_import, division, print_function)
 
 __metaclass__ = type
 
-__all__ = ["gssapi", "netaddr", "api", "ipalib_errors", "Env",
+__all__ = ["DEBUG_COMMAND_ALL", "DEBUG_COMMAND_LIST",
+           "DEBUG_COMMAND_COUNT", "DEBUG_COMMAND_BATCH",
+           "gssapi", "netaddr", "api", "ipalib_errors", "Env",
            "DEFAULT_CONFIG", "LDAP_GENERALIZED_TIME_FORMAT",
            "kinit_password", "kinit_keytab", "run", "DN", "VERSION",
            "paths", "tasks", "get_credentials_if_valid", "Encoding",
            "DNSName", "getargspec", "certificate_loader",
            "write_certificate_list", "boolean", "template_str",
-           "urlparse"]
+           "urlparse", "normalize_sshpubkey"]
+
+DEBUG_COMMAND_ALL = 0b1111
+# Print the while command list:
+DEBUG_COMMAND_LIST = 0b0001
+# Print the number of commands:
+DEBUG_COMMAND_COUNT = 0b0010
+# Print information about the batch slice size and currently executed batch
+# slice:
+DEBUG_COMMAND_BATCH = 0b0100
 
 import os
 # ansible-freeipa requires locale to be C, IPA requires utf-8.
@@ -44,6 +55,7 @@ import shutil
 import socket
 import base64
 import ast
+import time
 from datetime import datetime
 from contextlib import contextmanager
 from ansible.module_utils.basic import AnsibleModule
@@ -87,9 +99,13 @@ try:
     from ipalib.constants import DEFAULT_CONFIG, LDAP_GENERALIZED_TIME_FORMAT
 
     try:
-        from ipalib.install.kinit import kinit_password, kinit_keytab
+        from ipalib.kinit import kinit_password, kinit_keytab
     except ImportError:
-        from ipapython.ipautil import kinit_password, kinit_keytab
+        try:
+            from ipalib.install.kinit import kinit_password, kinit_keytab
+        except ImportError:
+            # pre 4.5.0
+            from ipapython.ipautil import kinit_password, kinit_keytab
     from ipapython.ipautil import run
     from ipapython.ipautil import template_str
     from ipapython.dn import DN
@@ -153,6 +169,8 @@ try:
     except ImportError:
         from ansible.module_utils.six.moves.urllib.parse import urlparse
 
+    from ipalib.util import normalize_sshpubkey
+
 except ImportError as _err:
     ANSIBLE_FREEIPA_MODULE_IMPORT_ERROR = str(_err)
 
@@ -481,7 +499,10 @@ def module_params_get(module, name, allow_empty_list_item=False):
     # Ansible issue https://github.com/ansible/ansible/issues/77108
     if isinstance(value, list):
         for val in value:
-            if isinstance(val, (str, unicode)) and not val:
+            if (
+                isinstance(val, (str, unicode))  # pylint: disable=W0012,E0606
+                and not val
+            ):
                 if not allow_empty_list_item:
                     module.fail_json(
                         msg="Parameter '%s' contains an empty string" %
@@ -1309,7 +1330,8 @@ class IPAAnsibleModule(AnsibleModule):
     def execute_ipa_commands(self, commands, result_handler=None,
                              exception_handler=None,
                              fail_on_member_errors=False,
-                             **handlers_user_args):
+                             batch=False, batch_slice_size=100, debug=False,
+                             keeponly=None, **handlers_user_args):
         """
         Execute IPA API commands from command list.
 
@@ -1326,6 +1348,16 @@ class IPAAnsibleModule(AnsibleModule):
             Returns True to continue to next command, else False
         fail_on_member_errors: bool
             Use default member error handler handler member_error_handler
+        batch: bool
+            Enable batch command use to speed up processing
+        batch_slice_size: integer
+            Maximum mumber of commands processed in a slice with the batch
+            command
+        keeponly: list of string
+            The attributes to keep in the results returned from the commands
+            Default: None (Keep all)
+        debug: integer
+            Enable debug output for the exection using DEBUG_COMMAND_*
         handlers_user_args: dict (user args mapping)
             The user args to pass to result_handler and exception_handler
             functions
@@ -1395,34 +1427,125 @@ class IPAAnsibleModule(AnsibleModule):
                 if "errors" in argspec.args:
                     handlers_user_args["errors"] = _errors
 
+        if debug & DEBUG_COMMAND_LIST:
+            self.tm_warn("commands: %s" % repr(commands))
+        if debug & DEBUG_COMMAND_COUNT:
+            self.tm_warn("#commands: %s" % len(commands))
+
+        # Turn off batch use for server context when it lacks the keeponly
+        # option as it lacks https://github.com/freeipa/freeipa/pull/7335
+        # This is an important fix about reporting errors in the batch
+        # (example: "no modifications to be performed") that results in
+        # aborted processing of the batch and an error about missing
+        # attribute principal. FreeIPA issue #9583
+        batch_has_keeponly = "keeponly" in api.Command.batch.options
+        if batch and api.env.in_server and not batch_has_keeponly:
+            self.debug(
+                "Turning off batch processing for batch missing keeponly")
+            batch = False
+
         changed = False
-        for name, command, args in commands:
-            try:
-                if name is None:
-                    result = self.ipa_command_no_name(command, args)
-                else:
-                    result = self.ipa_command(command, name, args)
+        if batch:
+            # batch processing
+            batch_args = []
+            for ci, (name, command, args) in enumerate(commands):
+                if len(batch_args) < batch_slice_size:
+                    batch_args.append({
+                        "method": command,
+                        "params": ([name], args)
+                    })
 
-                if "completed" in result:
-                    if result["completed"] > 0:
-                        changed = True
-                else:
-                    changed = True
-
-                # If result_handler is not None, call it with user args
-                # defined in **handlers_user_args
-                if result_handler is not None:
-                    result_handler(self, result, command, name, args,
-                                   **handlers_user_args)
-
-            except Exception as e:
-                if exception_handler is not None and \
-                   exception_handler(self, e, **handlers_user_args):
+                if len(batch_args) < batch_slice_size and \
+                   ci < len(commands) - 1:
+                    # fill in more commands untill batch slice size is reached
+                    # or final slice of commands
                     continue
-                self.fail_json(msg="%s: %s: %s" % (command, name, str(e)))
+
+                if debug & DEBUG_COMMAND_BATCH:
+                    self.tm_warn("batch %d (size %d/%d)" %
+                                 (ci / batch_slice_size, len(batch_args),
+                                  batch_slice_size))
+
+                # run the batch command
+                if batch_has_keeponly:
+                    result = api.Command.batch(batch_args, keeponly=keeponly)
+                else:
+                    result = api.Command.batch(batch_args)
+
+                if len(batch_args) != result["count"]:
+                    self.fail_json(
+                        "Result size %d does not match batch size %d" % (
+                            result["count"], len(batch_args)))
+                if result["count"] > 0:
+                    for ri, res in enumerate(result["results"]):
+                        _res = res.get("result", None)
+                        if not batch_has_keeponly and keeponly is not None \
+                           and isinstance(_res, dict):
+                            res["result"] = dict(
+                                filter(lambda x: x[0] in keeponly,
+                                       _res.items())
+                            )
+                        self.tm_warn("res: %s" % repr(res))
+
+                        if "error" not in res or res["error"] is None:
+                            if result_handler is not None:
+                                result_handler(
+                                    self, res,
+                                    batch_args[ri]["method"],
+                                    batch_args[ri]["params"][0][0],
+                                    batch_args[ri]["params"][1],
+                                    **handlers_user_args)
+                            changed = True
+                        else:
+                            _errors.append(
+                                "%s %s %s: %s" %
+                                (batch_args[ri]["method"],
+                                 repr(batch_args[ri]["params"][0][0]),
+                                 repr(batch_args[ri]["params"][1]),
+                                 res["error"]))
+                # clear batch command list (python2 compatible)
+                del batch_args[:]
+        else:
+            # no batch processing
+            for name, command, args in commands:
+                try:
+                    if name is None:
+                        result = self.ipa_command_no_name(command, args)
+                    else:
+                        result = self.ipa_command(command, name, args)
+
+                    if "completed" in result:
+                        if result["completed"] > 0:
+                            changed = True
+                    else:
+                        changed = True
+
+                    # Handle keeponly
+                    res = result.get("result", None)
+                    if keeponly is not None and isinstance(res, dict):
+                        result["result"] = dict(
+                            filter(lambda x: x[0] in keeponly, res.items())
+                        )
+
+                    # If result_handler is not None, call it with user args
+                    # defined in **handlers_user_args
+                    if result_handler is not None:
+                        result_handler(self, result, command, name, args,
+                                       **handlers_user_args)
+
+                except Exception as e:
+                    if exception_handler is not None and \
+                       exception_handler(self, e, **handlers_user_args):
+                        continue
+                    self.fail_json(msg="%s: %s: %s" % (command, name, str(e)))
 
         # Fail on errors from result_handler and exception_handler
         if len(_errors) > 0:
             self.fail_json(msg=", ".join(_errors))
 
         return changed
+
+    def tm_warn(self, warning):
+        ts = time.time()
+        # pylint: disable=super-with-arguments
+        super(IPAAnsibleModule, self).warn("%f %s" % (ts, warning))

plugins/modules/ipaautomember.py

@@ -450,6 +450,10 @@ def main():
         commands = []
 
         for name in names:
+            _type = None
+            inclusive_add, inclusive_del = [], []
+            exclusive_add, exclusive_del = [], []
+
             # Make sure automember rule exists
             res_find = find_automember(ansible_module, name, automember_type)
 
@@ -495,16 +499,12 @@ def main():
                             transform_conditions(inclusive),
                             res_find.get("automemberinclusiveregex", [])
                         )
-                    else:
-                        inclusive_add, inclusive_del = [], []
 
                     if exclusive is not None:
                         exclusive_add, exclusive_del = gen_add_del_lists(
                             transform_conditions(exclusive),
                             res_find.get("automemberexclusiveregex", [])
                         )
-                    else:
-                        exclusive_add, exclusive_del = [], []
 
                 elif action == "member":
                     if res_find is None:
@@ -512,9 +512,7 @@ def main():
                             msg="No automember '%s'" % name)
 
                     inclusive_add = transform_conditions(inclusive or [])
-                    inclusive_del = []
                     exclusive_add = transform_conditions(exclusive or [])
-                    exclusive_del = []
 
                 for _inclusive in inclusive_add:
                     key, regex = _inclusive.split("=", 1)

plugins/modules/ipadnsforwardzone.py

@@ -250,6 +250,8 @@ def main():
         operation = "add"
 
     invalid = []
+    wants_enable = False
+
     if state in ["enabled", "disabled"]:
         if action == "member":
             ansible_module.fail_json(

plugins/modules/ipadnsrecord.py

@@ -1605,6 +1605,8 @@ def main():
 
             res_find = find_dnsrecord(ansible_module, zone_name, name)
 
+            cmds = []
+
             if state == 'present':
                 cmds = define_commands_for_present_state(
                     ansible_module, zone_name, entry, res_find)

plugins/modules/ipagroup.py

@@ -663,7 +663,11 @@ def main():
 
                 check_parameters(ansible_module, state, action)
 
-            elif isinstance(group_name, (str, unicode)):
+            elif (
+                isinstance(
+                    group_name, (str, unicode)  # pylint: disable=W0012,E0606
+                )
+            ):
                 name = group_name
             else:
                 ansible_module.fail_json(msg="Group '%s' is not valid" %
@@ -900,7 +904,7 @@ def main():
 
         # Execute commands
         changed = ansible_module.execute_ipa_commands(
-            commands, fail_on_member_errors=True)
+            commands, batch=True, keeponly=[], fail_on_member_errors=True)
 
     # Done
 

plugins/modules/ipahost.py

@@ -509,7 +509,8 @@ host:
 
 from ansible.module_utils.ansible_freeipa_module import \
     IPAAnsibleModule, compare_args_ipa, gen_add_del_lists, \
-    encode_certificate, is_ipv4_addr, is_ipv6_addr, ipalib_errors
+    encode_certificate, is_ipv4_addr, is_ipv6_addr, ipalib_errors, \
+    gen_add_list, gen_intersection_list, normalize_sshpubkey
 from ansible.module_utils import six
 if six.PY3:
     unicode = str
@@ -533,6 +534,11 @@ def find_host(module, name):
     if certs is not None:
         _res["usercertificate"] = [encode_certificate(cert) for
                                    cert in certs]
+    # krbprincipalname is returned as ipapython.kerberos.Principal, convert
+    # to string
+    principals = _res.get("krbprincipalname")
+    if principals is not None:
+        _res["krbprincipalname"] = [str(princ) for princ in principals]
     return _res
 
 
@@ -676,8 +682,15 @@ def check_authind(module, auth_ind):
             "by your IPA version" % "','".join(_invalid))
 
 
+def convert_certificate(certificate):
+    if certificate is None:
+        return None
+
+    return [cert.strip() for cert in certificate]
+
+
 # pylint: disable=unused-argument
-def result_handler(module, result, command, name, args, errors, exit_args,
+def result_handler(module, result, command, name, args, exit_args,
                    single_host):
     if "random" in args and command in ["host_add", "host_mod"] \
        and "randompassword" in result["result"]:
@@ -688,41 +701,6 @@ def result_handler(module, result, command, name, args, errors, exit_args,
             exit_args.setdefault(name, {})["randompassword"] = \
                 result["result"]["randompassword"]
 
-    # All "already a member" and "not a member" failures in the
-    # result are ignored. All others are reported.
-    if "failed" in result and len(result["failed"]) > 0:
-        for item in result["failed"]:
-            failed_item = result["failed"][item]
-            for member_type in failed_item:
-                for member, failure in failed_item[member_type]:
-                    if "already a member" in failure \
-                       or "not a member" in failure:
-                        continue
-                    errors.append("%s: %s %s: %s" % (
-                        command, member_type, member, failure))
-
-
-# pylint: disable=unused-argument
-def exception_handler(module, ex, errors, exit_args, single_host):
-    msg = str(ex)
-    if "already contains" in msg \
-       or "does not contain" in msg:
-        return True
-
-    #  The canonical principal name may not be removed
-    if "equal to the canonical principal name must" in msg:
-        return True
-
-    # Host is already disabled, ignore error
-    if "This entry is already disabled" in msg:
-        return True
-
-    # Ignore no modification error.
-    if "no modifications to be performed" in msg:
-        return True
-
-    return False
-
 
 def main():
     host_spec = dict(
@@ -916,6 +894,11 @@ def main():
         auth_ind, requires_pre_auth, ok_as_delegate, ok_to_auth_as_delegate,
         force, reverse, ip_address, update_dns, update_password)
 
+    certificate = convert_certificate(certificate)
+
+    if sshpubkey is not None:
+        sshpubkey = [str(normalize_sshpubkey(key)) for key in sshpubkey]
+
     # Use hosts if names is None
     if hosts is not None:
         names = hosts
@@ -999,7 +982,15 @@ def main():
                     ok_to_auth_as_delegate, force, reverse, ip_address,
                     update_dns, update_password)
 
-            elif isinstance(host, (str, unicode)):
+                certificate = convert_certificate(certificate)
+
+                if sshpubkey is not None:
+                    sshpubkey = [str(normalize_sshpubkey(key)) for
+                                 key in sshpubkey]
+
+            elif (
+                isinstance(host, (str, unicode))  # pylint: disable=W0012,E0606
+            ):
                 name = host
             else:
                 ansible_module.fail_json(msg="Host '%s' is not valid" %
@@ -1074,6 +1065,17 @@ def main():
                            args["krbprincipalauthind"] == ['']:
                             del args["krbprincipalauthind"]
 
+                        # Ignore sshpubkey if it is empty (for resetting)
+                        # and not set for the host
+                        if "ipasshpubkey" not in res_find and \
+                           "ipasshpubkey" in args and \
+                           args["ipasshpubkey"] == []:
+                            del args["ipasshpubkey"]
+
+                        # Ignore updatedns if it is the only arg
+                        if "updatedns" in args and len(args) == 1:
+                            del args["updatedns"]
+
                         # For all settings is args, check if there are
                         # different settings in the find result.
                         # If yes: modify
@@ -1106,7 +1108,7 @@ def main():
                             gen_add_del_lists(managedby_host,
                                               res_find.get("managedby_host"))
                         principal_add, principal_del = gen_add_del_lists(
-                            principal, res_find.get("principal"))
+                            principal, res_find.get("krbprincipalname"))
                         # Principals are not returned as utf8 for IPA using
                         # python2 using host_show, therefore we need to
                         # convert the principals that we should remove.
@@ -1174,50 +1176,115 @@ def main():
                             gen_add_del_lists(
                                 dnsrecord_args.get("aaaarecord"),
                                 _dnsrec.get("aaaarecord"))
+                    else:
+                        certificate_add = certificate or []
+                        certificate_del = []
+                        managedby_host_add = managedby_host or []
+                        managedby_host_del = []
+                        principal_add = principal or []
+                        principal_del = []
+                        allow_create_keytab_user_add = \
+                            allow_create_keytab_user or []
+                        allow_create_keytab_user_del = []
+                        allow_create_keytab_group_add = \
+                            allow_create_keytab_group or []
+                        allow_create_keytab_group_del = []
+                        allow_create_keytab_host_add = \
+                            allow_create_keytab_host or []
+                        allow_create_keytab_host_del = []
+                        allow_create_keytab_hostgroup_add = \
+                            allow_create_keytab_hostgroup or []
+                        allow_create_keytab_hostgroup_del = []
+                        allow_retrieve_keytab_user_add = \
+                            allow_retrieve_keytab_user or []
+                        allow_retrieve_keytab_user_del = []
+                        allow_retrieve_keytab_group_add = \
+                            allow_retrieve_keytab_group or []
+                        allow_retrieve_keytab_group_del = []
+                        allow_retrieve_keytab_host_add = \
+                            allow_retrieve_keytab_host or []
+                        allow_retrieve_keytab_host_del = []
+                        allow_retrieve_keytab_hostgroup_add = \
+                            allow_retrieve_keytab_hostgroup or []
+                        allow_retrieve_keytab_hostgroup_del = []
+                        _dnsrec = res_find_dnsrecord or {}
+                        dnsrecord_a_add = gen_add_list(
+                            dnsrecord_args.get("arecord"),
+                            _dnsrec.get("arecord"))
+                        dnsrecord_a_del = []
+                        dnsrecord_aaaa_add = gen_add_list(
+                            dnsrecord_args.get("aaaarecord"),
+                            _dnsrec.get("aaaarecord"))
+                        dnsrecord_aaaa_del = []
 
                 else:
+                    # action member
                     if res_find is None:
                         ansible_module.fail_json(
                             msg="No host '%s'" % name)
 
-                if action != "host" or (action == "host" and res_find is None):
-                    certificate_add = certificate or []
+                    certificate_add = gen_add_list(
+                        certificate, res_find.get("usercertificate"))
                     certificate_del = []
-                    managedby_host_add = managedby_host or []
+                    managedby_host_add = gen_add_list(
+                        managedby_host, res_find.get("managedby_host"))
                     managedby_host_del = []
-                    principal_add = principal or []
+                    principal_add = gen_add_list(
+                        principal, res_find.get("krbprincipalname"))
                     principal_del = []
-                    allow_create_keytab_user_add = \
-                        allow_create_keytab_user or []
+                    allow_create_keytab_user_add = gen_add_list(
+                        allow_create_keytab_user,
+                        res_find.get(
+                            "ipaallowedtoperform_write_keys_user"))
                     allow_create_keytab_user_del = []
-                    allow_create_keytab_group_add = \
-                        allow_create_keytab_group or []
+                    allow_create_keytab_group_add = gen_add_list(
+                        allow_create_keytab_group,
+                        res_find.get(
+                            "ipaallowedtoperform_write_keys_group"))
                     allow_create_keytab_group_del = []
-                    allow_create_keytab_host_add = \
-                        allow_create_keytab_host or []
+                    allow_create_keytab_host_add = gen_add_list(
+                        allow_create_keytab_host,
+                        res_find.get(
+                            "ipaallowedtoperform_write_keys_host"))
                     allow_create_keytab_host_del = []
-                    allow_create_keytab_hostgroup_add = \
-                        allow_create_keytab_hostgroup or []
+                    allow_create_keytab_hostgroup_add = gen_add_list(
+                        allow_create_keytab_hostgroup,
+                        res_find.get(
+                            "ipaallowedtoperform_write_keys_hostgroup"))
                     allow_create_keytab_hostgroup_del = []
-                    allow_retrieve_keytab_user_add = \
-                        allow_retrieve_keytab_user or []
+                    allow_retrieve_keytab_user_add = gen_add_list(
+                        allow_retrieve_keytab_user,
+                        res_find.get(
+                            "ipaallowedtoperform_read_keys_user"))
                     allow_retrieve_keytab_user_del = []
-                    allow_retrieve_keytab_group_add = \
-                        allow_retrieve_keytab_group or []
+                    allow_retrieve_keytab_group_add = gen_add_list(
+                        allow_retrieve_keytab_group,
+                        res_find.get(
+                            "ipaallowedtoperform_read_keys_group"))
                     allow_retrieve_keytab_group_del = []
-                    allow_retrieve_keytab_host_add = \
-                        allow_retrieve_keytab_host or []
+                    allow_retrieve_keytab_host_add = gen_add_list(
+                        allow_retrieve_keytab_host,
+                        res_find.get(
+                            "ipaallowedtoperform_read_keys_host"))
                     allow_retrieve_keytab_host_del = []
-                    allow_retrieve_keytab_hostgroup_add = \
-                        allow_retrieve_keytab_hostgroup or []
+                    allow_retrieve_keytab_hostgroup_add = gen_add_list(
+                        allow_retrieve_keytab_hostgroup,
+                        res_find.get(
+                            "ipaallowedtoperform_read_keys_hostgroup"))
                     allow_retrieve_keytab_hostgroup_del = []
-                    dnsrecord_a_add = dnsrecord_args.get("arecord") or []
+                    _dnsrec = res_find_dnsrecord or {}
+                    dnsrecord_a_add = gen_add_list(
+                        dnsrecord_args.get("arecord"),
+                        _dnsrec.get("arecord"))
                     dnsrecord_a_del = []
-                    dnsrecord_aaaa_add = dnsrecord_args.get("aaaarecord") or []
+                    dnsrecord_aaaa_add = gen_add_list(
+                        dnsrecord_args.get("aaaarecord"),
+                        _dnsrec.get("aaaarecord"))
                     dnsrecord_aaaa_del = []
 
                 # Remove canonical principal from principal_del
                 canonical_principal = "host/" + name + "@" + server_realm
+                # canonical_principal is also in find_res["krbcanonicalname"]
                 if canonical_principal in principal_del and \
                    action == "host" and (principal is not None or
                                          canonical_principal not in principal):
@@ -1398,8 +1465,10 @@ def main():
                     # the removal of non-existing entries.
 
                     # Remove certificates
-                    if certificate is not None:
-                        for _certificate in certificate:
+                    certificate_del = gen_intersection_list(
+                        certificate, res_find.get("usercertificate"))
+                    if certificate_del is not None:
+                        for _certificate in certificate_del:
                             commands.append([name, "host_remove_cert",
                                              {
                                                  "usercertificate":
@@ -1412,8 +1481,10 @@ def main():
                     # the removal of non-existing entries.
 
                     # Remove managedby_hosts
-                    if managedby_host is not None:
-                        for _managedby_host in managedby_host:
+                    managedby_host_del = gen_intersection_list(
+                        managedby_host, res_find.get("managedby_host"))
+                    if managedby_host_del is not None:
+                        for _managedby_host in managedby_host_del:
                             commands.append([name, "host_remove_managedby",
                                              {
                                                  "host":
@@ -1426,8 +1497,10 @@ def main():
                     # the removal of non-existing entries.
 
                     # Remove principals
-                    if principal is not None:
-                        for _principal in principal:
+                    principal_del = gen_intersection_list(
+                        principal, res_find.get("krbprincipalname"))
+                    if principal_del is not None:
+                        for _principal in principal_del:
                             commands.append([name, "host_remove_principal",
                                              {
                                                  "krbprincipalname":
@@ -1435,60 +1508,86 @@ def main():
                                              }])
 
                     # Disallow create keytab
-                    if allow_create_keytab_user is not None or \
-                       allow_create_keytab_group is not None or \
-                       allow_create_keytab_host is not None or \
-                       allow_create_keytab_hostgroup is not None:
+                    allow_create_keytab_user_del = gen_intersection_list(
+                        allow_create_keytab_user,
+                        res_find.get("ipaallowedtoperform_write_keys_user"))
+                    allow_create_keytab_group_del = gen_intersection_list(
+                        allow_create_keytab_group,
+                        res_find.get("ipaallowedtoperform_write_keys_group"))
+                    allow_create_keytab_host_del = gen_intersection_list(
+                        allow_create_keytab_host,
+                        res_find.get("ipaallowedtoperform_write_keys_host"))
+                    allow_create_keytab_hostgroup_del = gen_intersection_list(
+                        allow_create_keytab_hostgroup,
+                        res_find.get(
+                            "ipaallowedtoperform_write_keys_hostgroup"))
+                    if len(allow_create_keytab_user_del) > 0 or \
+                       len(allow_create_keytab_group_del) > 0 or \
+                       len(allow_create_keytab_host_del) > 0 or \
+                       len(allow_create_keytab_hostgroup_del) > 0:
                         commands.append(
                             [name, "host_disallow_create_keytab",
                              {
-                                 "user": allow_create_keytab_user,
-                                 "group": allow_create_keytab_group,
-                                 "host": allow_create_keytab_host,
-                                 "hostgroup": allow_create_keytab_hostgroup,
+                                 "user": allow_create_keytab_user_del,
+                                 "group": allow_create_keytab_group_del,
+                                 "host": allow_create_keytab_host_del,
+                                 "hostgroup":
+                                 allow_create_keytab_hostgroup_del,
                              }])
 
                     # Disallow retrieve keytab
-                    if allow_retrieve_keytab_user is not None or \
-                       allow_retrieve_keytab_group is not None or \
-                       allow_retrieve_keytab_host is not None or \
-                       allow_retrieve_keytab_hostgroup is not None:
+                    allow_retrieve_keytab_user_del = gen_intersection_list(
+                        allow_retrieve_keytab_user,
+                        res_find.get("ipaallowedtoperform_read_keys_user"))
+                    allow_retrieve_keytab_group_del = gen_intersection_list(
+                        allow_retrieve_keytab_group,
+                        res_find.get("ipaallowedtoperform_read_keys_group"))
+                    allow_retrieve_keytab_host_del = gen_intersection_list(
+                        allow_retrieve_keytab_host,
+                        res_find.get("ipaallowedtoperform_read_keys_host"))
+                    allow_retrieve_keytab_hostgroup_del = \
+                        gen_intersection_list(
+                            allow_retrieve_keytab_hostgroup,
+                            res_find.get(
+                                "ipaallowedtoperform_read_keys_hostgroup"))
+                    if len(allow_retrieve_keytab_user_del) > 0 or \
+                       len(allow_retrieve_keytab_group_del) > 0 or \
+                       len(allow_retrieve_keytab_host_del) > 0 or \
+                       len(allow_retrieve_keytab_hostgroup_del) > 0:
                         commands.append(
                             [name, "host_disallow_retrieve_keytab",
                              {
-                                 "user": allow_retrieve_keytab_user,
-                                 "group": allow_retrieve_keytab_group,
-                                 "host": allow_retrieve_keytab_host,
-                                 "hostgroup": allow_retrieve_keytab_hostgroup,
+                                 "user": allow_retrieve_keytab_user_del,
+                                 "group": allow_retrieve_keytab_group_del,
+                                 "host": allow_retrieve_keytab_host_del,
+                                 "hostgroup":
+                                 allow_retrieve_keytab_hostgroup_del,
                              }])
 
-                    dnsrecord_args = gen_dnsrecord_args(ansible_module,
-                                                        ip_address, reverse)
+                    if res_find_dnsrecord is not None:
+                        dnsrecord_args = gen_dnsrecord_args(
+                            ansible_module, ip_address, reverse)
 
-                    # Remove arecord and aaaarecord from dnsrecord_args
-                    # if the record does not exits in res_find_dnsrecord
-                    # to prevent "DNS resource record not found" error
-                    if "arecord" in dnsrecord_args \
-                       and dnsrecord_args["arecord"] is not None \
-                       and len(dnsrecord_args["arecord"]) > 0 \
-                       and (res_find_dnsrecord is None
-                            or "arecord" not in res_find_dnsrecord):
-                        del dnsrecord_args["arecord"]
-                    if "aaaarecord" in dnsrecord_args \
-                       and dnsrecord_args["aaaarecord"] is not None \
-                       and len(dnsrecord_args["aaaarecord"]) > 0 \
-                       and (res_find_dnsrecord is None
-                            or "aaaarecord" not in res_find_dnsrecord):
-                        del dnsrecord_args["aaaarecord"]
+                        # Only keep a and aaaa recrords that are part
+                        # of res_find_dnsrecord.
+                        for _type in ["arecord", "aaaarecord"]:
+                            if _type in dnsrecord_args:
+                                recs = gen_intersection_list(
+                                    dnsrecord_args[_type],
+                                    res_find_dnsrecord.get(_type))
+                                if len(recs) > 0:
+                                    dnsrecord_args[_type] = recs
+                                else:
+                                    del dnsrecord_args[_type]
 
-                    if "arecord" in dnsrecord_args or \
-                       "aaaarecord" in dnsrecord_args:
-                        domain_name = name[name.find(".") + 1:]
-                        host_name = name[:name.find(".")]
-                        dnsrecord_args["idnsname"] = host_name
+                        if "arecord" in dnsrecord_args or \
+                           "aaaarecord" in dnsrecord_args:
+                            domain_name = name[name.find(".") + 1:]
+                            host_name = name[:name.find(".")]
+                            dnsrecord_args["idnsname"] = host_name
 
-                        commands.append([domain_name, "dnsrecord_del",
-                                         dnsrecord_args])
+                            commands.append([domain_name, "dnsrecord_del",
+                                             dnsrecord_args])
 
             elif state == "disabled":
                 if res_find is not None:
@@ -1504,7 +1603,7 @@ def main():
         # Execute commands
 
         changed = ansible_module.execute_ipa_commands(
-            commands, result_handler, exception_handler,
+            commands, result_handler, batch=True, keeponly=["randompassword"],
             exit_args=exit_args, single_host=hosts is None)
 
     # Done

plugins/modules/iparole.py

@@ -293,7 +293,7 @@ def result_get_value_lowercase(res_find, key, default=None):
     if existing is not None:
         if isinstance(existing, (list, tuple)):
             existing = [to_text(item).lower() for item in existing]
-        if isinstance(existing, (str, unicode)):
+        if isinstance(existing, (str, unicode)):  # pylint: disable=W0012,E0606
             existing = existing.lower()
     else:
         existing = default

plugins/modules/ipaservice.py

@@ -693,7 +693,11 @@ def main():
 
                 delete_continue = service.get("delete_continue")
 
-            elif isinstance(service, (str, unicode)):
+            elif (
+                isinstance(
+                    service, (str, unicode)  # pylint: disable=W0012,E0606
+                )
+            ):
                 name = service
             else:
                 ansible_module.fail_json(msg="Service '%s' is not valid" %
@@ -840,7 +844,9 @@ def main():
             elif state == "absent":
                 if action == "service":
                     if res_find is not None:
-                        args = {'continue': delete_continue}
+                        args = {}
+                        if delete_continue is not None:
+                            args['continue'] = delete_continue
                         commands.append([name, 'service_del', args])
 
                 elif action == "member":
@@ -929,7 +935,7 @@ def main():
 
         # Execute commands
         changed = ansible_module.execute_ipa_commands(
-            commands, fail_on_member_errors=True)
+            commands, batch=True, keeponly=[], fail_on_member_errors=True)
 
     # Done
     ansible_module.exit_json(changed=changed, **exit_args)

plugins/modules/ipauser.py

@@ -741,7 +741,7 @@ user:
 from ansible.module_utils.ansible_freeipa_module import \
     IPAAnsibleModule, compare_args_ipa, gen_add_del_lists, date_format, \
     encode_certificate, load_cert_from_str, DN_x500_text, to_text, \
-    ipalib_errors
+    ipalib_errors, gen_add_list, gen_intersection_list
 from ansible.module_utils import six
 if six.PY3:
     unicode = str
@@ -961,6 +961,13 @@ def extend_emails(email, default_email_domain):
     return email
 
 
+def convert_certificate(certificate):
+    if certificate is None:
+        return None
+
+    return [cert.strip() for cert in certificate]
+
+
 def convert_certmapdata(certmapdata):
     if certmapdata is None:
         return None
@@ -1006,9 +1013,8 @@ def gen_certmapdata_args(certmapdata):
 
 
 # pylint: disable=unused-argument
-def result_handler(module, result, command, name, args, errors, exit_args,
-                   single_user):
-
+def result_handler(module, result, command, name, args, exit_args,
+                   errors, single_user):
     if "random" in args and command in ["user_add", "user_mod"] \
        and "randompassword" in result["result"]:
         if single_user:
@@ -1018,31 +1024,8 @@ def result_handler(module, result, command, name, args, errors, exit_args,
             exit_args.setdefault(name, {})["randompassword"] = \
                 result["result"]["randompassword"]
 
-    # Get all errors
-    # All "already a member" and "not a member" failures in the
-    # result are ignored. All others are reported.
-    if "failed" in result and len(result["failed"]) > 0:
-        for item in result["failed"]:
-            failed_item = result["failed"][item]
-            for member_type in failed_item:
-                for member, failure in failed_item[member_type]:
-                    if "already a member" in failure \
-                       or "not a member" in failure:
-                        continue
-                    errors.append("%s: %s %s: %s" % (
-                        command, member_type, member, failure))
-
-
-# pylint: disable=unused-argument
-def exception_handler(module, ex, errors, exit_args, single_user):
-    msg = str(ex)
-    if "already contains" in msg \
-       or "does not contain" in msg:
-        return True
-    #  The canonical principal name may not be removed
-    if "equal to the canonical principal name must" in msg:
-        return True
-    return False
+    IPAAnsibleModule.member_error_handler(module, result, command, name, args,
+                                          errors)
 
 
 def main():
@@ -1277,6 +1260,7 @@ def main():
             preserve, update_password, smb_logon_script, smb_profile_path,
             smb_home_dir, smb_home_drive, idp, idp_user_id, rename,
         )
+        certificate = convert_certificate(certificate)
         certmapdata = convert_certmapdata(certmapdata)
 
     # Init
@@ -1387,6 +1371,7 @@ def main():
                     update_password, smb_logon_script, smb_profile_path,
                     smb_home_dir, smb_home_drive, idp, idp_user_id, rename,
                 )
+                certificate = convert_certificate(certificate)
                 certmapdata = convert_certmapdata(certmapdata)
 
                 # Check API specific parameters
@@ -1397,7 +1382,11 @@ def main():
 
                 email = extend_emails(email, default_email_domain)
 
-            elif isinstance(user, (str, unicode)):
+            elif (
+                isinstance(
+                    user, (str, unicode)  # pylint: disable=W0012,E0606
+                )
+            ):
                 name = user
             else:
                 ansible_module.fail_json(msg="User '%s' is not valid" %
@@ -1646,10 +1635,12 @@ def main():
                             msg="No user '%s'" % name)
 
                     # Ensure managers are present
-                    if manager is not None and len(manager) > 0:
+                    manager_add = gen_add_list(
+                        manager, res_find.get("manager"))
+                    if manager_add is not None and len(manager_add) > 0:
                         commands.append([name, "user_add_manager",
                                          {
-                                             "user": manager,
+                                             "user": manager_add,
                                          }])
 
                     # Principals need to be added and removed one by one,
@@ -1658,8 +1649,10 @@ def main():
                     # the removal of non-existing entries.
 
                     # Ensure principals are present
-                    if principal is not None and len(principal) > 0:
-                        for _principal in principal:
+                    principal_add = gen_add_list(
+                        principal, res_find.get("krbprincipalname"))
+                    if principal_add is not None and len(principal_add) > 0:
+                        for _principal in principal_add:
                             commands.append([name, "user_add_principal",
                                              {
                                                  "krbprincipalname":
@@ -1672,8 +1665,11 @@ def main():
                     # the removal of non-existing entries.
 
                     # Ensure certificates are present
-                    if certificate is not None and len(certificate) > 0:
-                        for _certificate in certificate:
+                    certificate_add = gen_add_list(
+                        certificate, res_find.get("usercertificate"))
+                    if certificate_add is not None and \
+                       len(certificate_add) > 0:
+                        for _certificate in certificate_add:
                             commands.append([name, "user_add_cert",
                                              {
                                                  "usercertificate":
@@ -1685,8 +1681,11 @@ def main():
                     # one reliably (https://pagure.io/freeipa/issue/8097)
 
                     # Ensure certmapdata are present
-                    if certmapdata is not None and len(certmapdata) > 0:
-                        for _data in certmapdata:
+                    certmapdata_add = gen_add_list(
+                        certmapdata, res_find.get("ipacertmapdata"))
+                    if certmapdata_add is not None and \
+                       len(certmapdata_add) > 0:
+                        for _data in certmapdata_add:
                             commands.append([name, "user_add_certmapdata",
                                              gen_certmapdata_args(_data)])
 
@@ -1707,10 +1706,12 @@ def main():
                             msg="No user '%s'" % name)
 
                     # Ensure managers are absent
-                    if manager is not None and len(manager) > 0:
+                    manager_del = gen_intersection_list(
+                        manager, res_find.get("manager"))
+                    if manager_del is not None and len(manager_del) > 0:
                         commands.append([name, "user_remove_manager",
                                          {
-                                             "user": manager,
+                                             "user": manager_del,
                                          }])
 
                     # Principals need to be added and removed one by one,
@@ -1719,10 +1720,12 @@ def main():
                     # the removal of non-existing entries.
 
                     # Ensure principals are absent
-                    if principal is not None and len(principal) > 0:
+                    principal_del = gen_intersection_list(
+                        principal, res_find.get("krbprincipalname"))
+                    if principal_del is not None and len(principal_del) > 0:
                         commands.append([name, "user_remove_principal",
                                          {
-                                             "krbprincipalname": principal,
+                                             "krbprincipalname": principal_del,
                                          }])
 
                     # Certificates need to be added and removed one by one,
@@ -1731,8 +1734,11 @@ def main():
                     # the removal of non-existing entries.
 
                     # Ensure certificates are absent
-                    if certificate is not None and len(certificate) > 0:
-                        for _certificate in certificate:
+                    certificate_del = gen_intersection_list(
+                        certificate, res_find.get("usercertificate"))
+                    if certificate_del is not None and \
+                       len(certificate_del) > 0:
+                        for _certificate in certificate_del:
                             commands.append([name, "user_remove_cert",
                                              {
                                                  "usercertificate":
@@ -1744,10 +1750,13 @@ def main():
                     # one reliably (https://pagure.io/freeipa/issue/8097)
 
                     # Ensure certmapdata are absent
-                    if certmapdata is not None and len(certmapdata) > 0:
+                    certmapdata_del = gen_intersection_list(
+                        certmapdata, res_find.get("ipacertmapdata"))
+                    if certmapdata_del is not None and \
+                       len(certmapdata_del) > 0:
                         # Using issuer and subject can only be done one by
                         # one reliably (https://pagure.io/freeipa/issue/8097)
-                        for _data in certmapdata:
+                        for _data in certmapdata_del:
                             commands.append([name, "user_remove_certmapdata",
                                              gen_certmapdata_args(_data)])
             elif state == "undeleted":
@@ -1791,7 +1800,7 @@ def main():
         # Execute commands
 
         changed = ansible_module.execute_ipa_commands(
-            commands, result_handler, exception_handler,
+            commands, result_handler, batch=True, keeponly=["randompassword"],
             exit_args=exit_args, single_user=users is None)
 
     # Done

requirements-dev.txt

@@ -1,10 +1,10 @@
 -r requirements-tests.txt
 ipdb==0.13.4
 pre-commit==2.20.0
-flake8==6.0.0
+flake8==7.0.0
 flake8-bugbear
-pylint==2.17.2
+pylint>=3.2
 wrapt==1.14.1
 pydocstyle==6.3.0
-yamllint==1.32.0
-ansible-lint >= 6.22
+yamllint==1.35.1
+ansible-lint>=24.5.0

roles/ipaclient/library/ipaclient_configure_dns_resolver.py

@@ -152,8 +152,10 @@ def configure_dns_resolver(nameservers, searchdomains, fstore=None):
     if not searchdomains or not isinstance(searchdomains, list):
         raise AssertionError("searchdomains must be of type list")
 
+    changed = False
     if fstore is not None and not fstore.has_file(paths.RESOLV_CONF):
         fstore.backup_file(paths.RESOLV_CONF)
+        changed = True
 
     resolve1_enabled = detect_resolve1_resolv_conf()
     if "NetworkManager" not in services.knownservices:
@@ -192,6 +194,7 @@ def configure_dns_resolver(nameservers, searchdomains, fstore=None):
                 sdrd_service = services.service("systemd-resolved.service")
             if sdrd_service.is_enabled():
                 sdrd_service.reload_or_restart()
+            changed = True
 
     # Then configure NetworkManager or resolve.conf
     if nm_service.is_enabled():
@@ -217,6 +220,7 @@ def configure_dns_resolver(nameservers, searchdomains, fstore=None):
                 outf.write(cfg)
             # reload NetworkManager
             nm_service.reload_or_restart()
+            changed = True
 
     # Configure resolv.conf if NetworkManager and systemd-resoled are not
     # enabled
@@ -231,6 +235,9 @@ def configure_dns_resolver(nameservers, searchdomains, fstore=None):
             cfg.append("nameserver %s" % nameserver)
         with open(paths.RESOLV_CONF, 'w') as outf:
             outf.write('\n'.join(cfg))
+        changed = True
+
+    return changed
 
 
 def unconfigure_dns_resolver(fstore=None):
@@ -239,8 +246,11 @@ def unconfigure_dns_resolver(fstore=None):
 
     :param fstore: optional file store for resolv.conf restore
     """
+    changed = False
+
     if fstore is not None and fstore.has_file(paths.RESOLV_CONF):
         fstore.restore_file(paths.RESOLV_CONF)
+        changed = True
 
     if os.path.isfile(NETWORK_MANAGER_IPA_CONF):
         os.unlink(NETWORK_MANAGER_IPA_CONF)
@@ -252,6 +262,7 @@ def unconfigure_dns_resolver(fstore=None):
             nm_service = services.knownservices['NetworkManager']
         if nm_service.is_enabled():
             nm_service.reload_or_restart()
+        changed = True
 
     if os.path.isfile(SYSTEMD_RESOLVED_IPA_CONF):
         os.unlink(SYSTEMD_RESOLVED_IPA_CONF)
@@ -261,6 +272,9 @@ def unconfigure_dns_resolver(fstore=None):
             sdrd_service = services.service("systemd-resolved.service")
         if sdrd_service.is_enabled():
             sdrd_service.reload_or_restart()
+        changed = True
+
+    return changed
 
 
 def main():
@@ -308,11 +322,12 @@ def main():
     fstore = sysrestore.FileStore(paths.IPA_CLIENT_SYSRESTORE)
 
     if state == "present":
-        configure_dns_resolver(nameservers, searchdomains, fstore)
+        changed = configure_dns_resolver(nameservers,
+                                         searchdomains, fstore)
     else:
-        unconfigure_dns_resolver(fstore)
+        changed = unconfigure_dns_resolver(fstore)
 
-    module.exit_json(changed=True)
+    module.exit_json(changed=changed)
 
 
 if __name__ == '__main__':

roles/ipaclient/library/ipaclient_get_otp.py

@@ -89,9 +89,13 @@ try:
     from ipapython.ipautil import run
     from ipalib.constants import DEFAULT_CONFIG
     try:
-        from ipalib.install.kinit import kinit_password, kinit_keytab
+        from ipalib.kinit import kinit_password, kinit_keytab
     except ImportError:
-        from ipapython.ipautil import kinit_password, kinit_keytab
+        try:
+            from ipalib.install.kinit import kinit_password, kinit_keytab
+        except ImportError:
+            # pre 4.5.0
+            from ipapython.ipautil import kinit_password, kinit_keytab
 except ImportError as _err:
     MODULE_IMPORT_ERROR = str(_err)
 else:

roles/ipaclient/module_utils/ansible_ipa_client.py

@@ -173,9 +173,13 @@ try:
             ipa_generate_password
         from ipapython.dn import DN
         try:
-            from ipalib.install.kinit import kinit_keytab, kinit_password
+            from ipalib.kinit import kinit_password, kinit_keytab
         except ImportError:
-            from ipapython.ipautil import kinit_keytab, kinit_password
+            try:
+                from ipalib.install.kinit import kinit_keytab, kinit_password
+            except ImportError:
+                # pre 4.5.0
+                from ipapython.ipautil import kinit_keytab, kinit_password
         from ipapython.ipa_log_manager import standard_logging_setup
         from gssapi.exceptions import GSSError
         try:

roles/ipareplica/library/ipareplica_add_to_ipaservers.py

@@ -139,7 +139,7 @@ def main():
         conn.connect(ccache=installer._ccache)
         remote_api.Command['hostgroup_add_member'](
             u'ipaservers',
-            host=[unicode(api.env.host)],
+            host=[unicode(api.env.host)],  # pylint: disable=W0012,E0606
         )
     finally:
         if conn.isconnected():

roles/ipareplica/library/ipareplica_prepare.py

@@ -658,7 +658,7 @@ def main():
         # Check authorization
         result = remote_api.Command['hostgroup_find'](
             cn=u'ipaservers',
-            host=[unicode(api.env.host)]
+            host=[unicode(api.env.host)]  # pylint: disable=W0012,E0606
         )['result']
         add_to_ipaservers = not result
 

roles/ipareplica/module_utils/ansible_ipa_replica.py

@@ -104,7 +104,10 @@ try:
         from ipaclient.install.ipachangeconf import IPAChangeConf
         from ipalib.install import certstore, sysrestore
         from ipapython.ipautil import ipa_generate_password
-        from ipalib.install.kinit import kinit_keytab
+        try:
+            from ipalib.kinit import kinit_keytab
+        except ImportError:
+            from ipalib.install.kinit import kinit_keytab
         from ipapython import ipaldap, ipautil, kernel_keyring
         from ipapython.certdb import IPA_CA_TRUST_FLAGS, \
             EXTERNAL_CA_TRUST_FLAGS

roles/ipaserver/library/ipaserver_get_connected_server.py

@@ -77,9 +77,13 @@ try:
     from ipapython.ipautil import run
     from ipalib.constants import DEFAULT_CONFIG
     try:
-        from ipalib.install.kinit import kinit_password
+        from ipalib.kinit import kinit_password
     except ImportError:
-        from ipapython.ipautil import kinit_password
+        try:
+            from ipalib.install.kinit import kinit_password
+        except ImportError:
+            # pre 4.5.0
+            from ipapython.ipautil import kinit_password
 except ImportError as _err:
     MODULE_IMPORT_ERROR = str(_err)
 else:

roles/ipaserver/library/ipaserver_prepare.py

@@ -226,7 +226,8 @@ from ansible.module_utils.ansible_ipa_server import (
     redirect_stdout, adtrust, api, default_subject_base,
     default_ca_subject_dn, ipautil, installutils, ca, kra, dns,
     get_server_ip_address, no_matching_interface_for_ip_address_warning,
-    services, logger, tasks, update_hosts_file, ScriptError
+    services, logger, tasks, update_hosts_file, ScriptError, IPAChangeConf,
+    realm_to_ldapi_uri
 )
 
 
@@ -365,6 +366,11 @@ def main():
     fstore = sysrestore.FileStore(paths.SYSRESTORE)
     sstore = sysrestore.StateFile(paths.SYSRESTORE)
 
+    domain_name = options.domain_name
+    realm_name = options.realm_name
+    host_name = options.host_name
+    setup_ca = options.setup_ca
+
     # subject_base
     if not options.subject_base:
         options.subject_base = str(default_subject_base(options.realm_name))
@@ -391,27 +397,68 @@ def main():
 
         # Create the management framework config file and finalize api
         target_fname = paths.IPA_DEFAULT_CONF
-        # pylint: disable=invalid-name, consider-using-with
-        fd = open(target_fname, "w")
-        fd.write("[global]\n")
-        fd.write("host=%s\n" % options.host_name)
-        fd.write("basedn=%s\n" % ipautil.realm_to_suffix(options.realm_name))
-        fd.write("realm=%s\n" % options.realm_name)
-        fd.write("domain=%s\n" % options.domain_name)
-        fd.write("xmlrpc_uri=https://%s/ipa/xml\n" %
-                 ipautil.format_netloc(options.host_name))
-        fd.write("ldap_uri=ldapi://%%2fvar%%2frun%%2fslapd-%s.socket\n" %
-                 installutils.realm_to_serverid(options.realm_name))
-        if options.setup_ca:
-            fd.write("enable_ra=True\n")
-            fd.write("ra_plugin=dogtag\n")
-            fd.write("dogtag_version=10\n")
+        if realm_to_ldapi_uri is not None:
+            ipaconf = IPAChangeConf("IPA Server Install")
+            ipaconf.setOptionAssignment(" = ")
+            ipaconf.setSectionNameDelimiters(("[", "]"))
+
+            xmlrpc_uri = 'https://{0}/ipa/xml'.format(
+                ipautil.format_netloc(host_name))
+            ldapi_uri = realm_to_ldapi_uri(realm_name)
+
+            # [global] section
+            gopts = [
+                ipaconf.setOption('host', host_name),
+                ipaconf.setOption('basedn',
+                                  ipautil.realm_to_suffix(realm_name)),
+                ipaconf.setOption('realm', realm_name),
+                ipaconf.setOption('domain', domain_name),
+                ipaconf.setOption('xmlrpc_uri', xmlrpc_uri),
+                ipaconf.setOption('ldap_uri', ldapi_uri),
+                ipaconf.setOption('mode', 'production')
+            ]
+
+            if setup_ca:
+                gopts.extend([
+                    ipaconf.setOption('enable_ra', 'True'),
+                    ipaconf.setOption('ra_plugin', 'dogtag'),
+                    ipaconf.setOption('dogtag_version', '10')
+                ])
+            else:
+                gopts.extend([
+                    ipaconf.setOption('enable_ra', 'False'),
+                    ipaconf.setOption('ra_plugin', 'None')
+                ])
+
+            opts = [
+                ipaconf.setSection('global', gopts),
+                {'name': 'empty', 'type': 'empty'}
+            ]
+
+            ipaconf.newConf(target_fname, opts)
         else:
-            fd.write("enable_ra=False\n")
-            fd.write("ra_plugin=none\n")
-        fd.write("mode=production\n")
-        fd.close()
-        # pylint: enable=invalid-name, consider-using-with
+            # pylint: disable=invalid-name, consider-using-with
+            fd = open(target_fname, "w")
+            fd.write("[global]\n")
+            fd.write("host=%s\n" % options.host_name)
+            fd.write("basedn=%s\n" % ipautil.realm_to_suffix(
+                options.realm_name))
+            fd.write("realm=%s\n" % options.realm_name)
+            fd.write("domain=%s\n" % options.domain_name)
+            fd.write("xmlrpc_uri=https://%s/ipa/xml\n" %
+                     ipautil.format_netloc(options.host_name))
+            fd.write("ldap_uri=ldapi://%%2fvar%%2frun%%2fslapd-%s.socket\n" %
+                     installutils.realm_to_serverid(options.realm_name))
+            if options.setup_ca:
+                fd.write("enable_ra=True\n")
+                fd.write("ra_plugin=dogtag\n")
+                fd.write("dogtag_version=10\n")
+            else:
+                fd.write("enable_ra=False\n")
+                fd.write("ra_plugin=none\n")
+            fd.write("mode=production\n")
+            fd.close()
+            # pylint: enable=invalid-name, consider-using-with
 
         # Must be readable for everyone
         os.chmod(target_fname, 0o644)

roles/ipaserver/library/ipaserver_setup_ca.py

@@ -354,7 +354,7 @@ def main():
                       options.no_hbac_allow, options._dirsrv_pkcs12_info,
                       options.no_pkinit)
 
-    # setup CA ##############################################################
+    # setup custodia ########################################################
 
     if hasattr(custodiainstance, "get_custodia_instance"):
         if hasattr(custodiainstance.CustodiaModes, "FIRST_MASTER"):
@@ -362,9 +362,14 @@ def main():
         else:
             mode = custodiainstance.CustodiaModes.MASTER_PEER
         custodia = custodiainstance.get_custodia_instance(options, mode)
-        custodia.set_output(ansible_log)
-        with redirect_stdout(ansible_log):
-            custodia.create_instance()
+    else:
+        custodia = custodiainstance.CustodiaInstance(options.host_name,
+                                                     options.realm_name)
+    custodia.set_output(ansible_log)
+    with redirect_stdout(ansible_log):
+        custodia.create_instance()
+
+    # setup CA ##############################################################
 
     if options.setup_ca:
         if not options.external_cert_files and options.external_ca:

roles/ipaserver/library/ipaserver_setup_custodia.py

@@ -1,118 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Authors:
-#   Thomas Woerner <twoerner@redhat.com>
-#
-# Based on ipa-client-install code
-#
-# Copyright (C) 2017-2022  Red Hat
-# see file 'COPYING' for use and warranty information
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-__metaclass__ = type
-
-ANSIBLE_METADATA = {
-    'metadata_version': '1.0',
-    'supported_by': 'community',
-    'status': ['preview'],
-}
-
-DOCUMENTATION = '''
----
-module: ipaserver_setup_custodia
-short_description: Setup custodia
-description: Setup custodia
-options:
-  realm:
-    description: Kerberos realm name of the IPA deployment
-    type: str
-    required: yes
-  hostname:
-    description: Fully qualified name of this host
-    type: str
-    required: no
-  setup_ca:
-    description: Configure a dogtag CA
-    type: bool
-    default: no
-    required: no
-author:
-    - Thomas Woerner (@t-woerner)
-'''
-
-EXAMPLES = '''
-'''
-
-RETURN = '''
-'''
-
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.ansible_ipa_server import (
-    check_imports, setup_logging, AnsibleModuleLog, options,
-    api_Backend_ldap2,
-    custodiainstance, redirect_stdout
-)
-
-
-def main():
-    ansible_module = AnsibleModule(
-        argument_spec=dict(
-            # basic
-            realm=dict(required=True, type='str'),
-            hostname=dict(required=False, type='str'),
-            setup_ca=dict(required=False, type='bool', default=False),
-        ),
-    )
-
-    ansible_module._ansible_debug = True
-    check_imports(ansible_module)
-    setup_logging()
-    ansible_log = AnsibleModuleLog(ansible_module)
-
-    # set values ############################################################
-
-    options.realm_name = ansible_module.params.get('realm')
-    options.host_name = ansible_module.params.get('hostname')
-    options.setup_ca = ansible_module.params.get('setup_ca')
-    options.promote = False
-
-    # init ##################################################################
-
-    api_Backend_ldap2(options.host_name, options.setup_ca, connect=True)
-
-    # setup custodia ########################################################
-
-    if hasattr(custodiainstance, "get_custodia_instance"):
-        if hasattr(custodiainstance.CustodiaModes, "FIRST_MASTER"):
-            mode = custodiainstance.CustodiaModes.FIRST_MASTER
-        else:
-            mode = custodiainstance.CustodiaModes.MASTER_PEER
-        custodia = custodiainstance.get_custodia_instance(options, mode)
-    else:
-        custodia = custodiainstance.CustodiaInstance(options.host_name,
-                                                     options.realm_name)
-    custodia.set_output(ansible_log)
-    with redirect_stdout(ansible_log):
-        custodia.create_instance()
-
-    # done ##################################################################
-
-    ansible_module.exit_json(changed=True)
-
-
-if __name__ == '__main__':
-    main()

roles/ipaserver/library/ipaserver_test.py

@@ -1171,7 +1171,7 @@ def main():
         changed=False,
         ipa_python_version=IPA_PYTHON_VERSION,
         # basic
-        domain=options.domain_name,
+        domain=domain_name,
         realm=realm_name,
         hostname=host_name,
         _hostname_overridden=bool(options.host_name),

roles/ipaserver/module_utils/ansible_ipa_server.py

@@ -44,7 +44,7 @@ __all__ = ["IPAChangeConf", "certmonger", "sysrestore", "root_logger",
            "check_available_memory", "getargspec", "get_min_idstart",
            "paths", "api", "ipautil", "adtrust_imported", "NUM_VERSION",
            "time_service", "kra_imported", "dsinstance", "IPA_PYTHON_VERSION",
-           "NUM_VERSION", "SerialNumber"]
+           "NUM_VERSION", "SerialNumber", "realm_to_ldapi_uri"]
 
 import sys
 import logging
@@ -121,6 +121,10 @@ try:
         )
         from ipapython.dnsutil import check_zone_overlap
         from ipapython.dn import DN
+        try:
+            from ipapython.ipaldap import realm_to_ldapi_uri
+        except ImportError:
+            realm_to_ldapi_uri = None
         try:
             from ipaclient.install import timeconf
             from ipaclient.install.client import sync_time

roles/ipaserver/tasks/install.yml

@@ -267,12 +267,6 @@
       idmax: "{{ result_ipaserver_test.idmax }}"
       _pkinit_pkcs12_info: "{{ result_ipaserver_test._pkinit_pkcs12_info if result_ipaserver_test._pkinit_pkcs12_info != None else omit }}"
 
-  - name: Install - Setup custodia
-    ipaserver_setup_custodia:
-      realm: "{{ result_ipaserver_test.realm }}"
-      hostname: "{{ result_ipaserver_test.hostname }}"
-      setup_ca: "{{ result_ipaserver_test.setup_ca }}"
-
   - name: Install - Setup CA
     ipaserver_setup_ca:
       dm_password: "{{ ipadm_password }}"

tests/ca-less/certificates/extensions.conf

@@ -0,0 +1,7 @@
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+authorityKeyIdentifier = keyid,issuer
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = ${ENV::HOST_FQDN}

tests/ca-less/certificates/pkinit-extensions.conf

@@ -0,0 +1,19 @@
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation,digitalSignature,keyEncipherment,keyAgreement
+extendedKeyUsage = 1.3.6.1.5.2.3.5
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+issuerAltName = issuer:copy
+subjectAltName = otherName:1.3.6.1.5.2.2;SEQUENCE:kdc_princ_name
+
+[kdc_princ_name]
+realm = EXP:0,GeneralString:${ENV::REALM_NAME}
+principal_name = EXP:1,SEQUENCE:kdc_principal_seq
+
+[kdc_principal_seq]
+name_type = EXP:0,INTEGER:1
+name_string = EXP:1,SEQUENCE:kdc_principals
+
+[kdc_principals]
+princ1 = GeneralString:krbtgt
+princ2 = GeneralString:${ENV::REALM_NAME}

tests/ca-less/certificates/pkinit/extensions.conf

@@ -1,20 +0,0 @@
-[kdc_cert]
-basicConstraints=CA:FALSE
-keyUsage=nonRepudiation,digitalSignature,keyEncipherment,keyAgreement
-extendedKeyUsage=1.3.6.1.5.2.3.5
-subjectKeyIdentifier=hash
-authorityKeyIdentifier=keyid,issuer
-issuerAltName=issuer:copy
-subjectAltName=otherName:1.3.6.1.5.2.2;SEQUENCE:kdc_princ_name
-
-[kdc_princ_name]
-realm=EXP:0,GeneralString:${ENV::REALM}
-principal_name=EXP:1,SEQUENCE:kdc_principal_seq
-
-[kdc_principal_seq]
-name_type=EXP:0,INTEGER:1
-name_string=EXP:1,SEQUENCE:kdc_principals
-
-[kdc_principals]
-princ1=GeneralString:krbtgt
-princ2=GeneralString:${ENV::REALM}

tests/ca-less/clean_up_certificates.yml

@@ -7,9 +7,6 @@
   - name: Run generate-certificates.sh
     ansible.builtin.command: >
       /bin/bash
-      generate-certificates.sh delete "{{ item }}"
+      generate-certificates.sh cleanup
     args:
       chdir: "{{ playbook_dir }}"
-    with_items:
-      - "{{ groups.ipaserver[0] }}"
-      - "{{ groups.ipareplicas[0] }}"

tests/ca-less/generate-certificates.sh

@@ -1,153 +1,177 @@
 #!/usr/bin/env bash
 
-ROOT_CA_DIR="certificates/root-ca"
-DIRSRV_CERTS_DIR="certificates/dirsrv"
-HTTPD_CERTS_DIR="certificates/httpd"
-PKINIT_CERTS_DIR="certificates/pkinit"
+CERTIFICATES="certificates"
+ROOT_CA_DIR="${CERTIFICATES}/root-ca"
+DIRSRV_CERTS_DIR="${CERTIFICATES}/dirsrv"
+HTTPD_CERTS_DIR="${CERTIFICATES}/httpd"
+PKINIT_CERTS_DIR="${CERTIFICATES}/pkinit"
+EXTENSIONS_CONF="${CERTIFICATES}/extensions.conf"
+PKINIT_EXTENSIONS_CONF="${CERTIFICATES}/pkinit-extensions.conf"
 PKCS12_PASSWORD="SomePKCS12password"
 
-# generate_ipa_pkcs12_certificate \
-#    $cert_name $ipa_fqdn $certs_dir $root_ca_cert $root_ca_private_key extensions_file extensions_name
-function generate_ipa_pkcs12_certificate {
+# create_ca \
+#    $domain_name
+function create_ca {
 
-    cert_name=$1
-    ipa_fqdn=$2
-    certs_dir=$3
-    root_ca_cert=$4
-    root_ca_private_key=$5
-    extensions_file=$6
-    extensions_name=$7
-
-    # Generate CSR and private key
-    openssl req -new -newkey rsa:4096 -nodes \
-        -subj "/C=US/ST=Test/L=Testing/O=Default/CN=${ipa_fqdn}" \
-        -keyout "${certs_dir}/private.key" \
-        -out "${certs_dir}/request.csr"
-
-    # Sign CSR to generate PEM certificate
-    if [ -z "${extensions_file}" ]; then
-        openssl x509 -req -days 365 -sha256 \
-            -CAcreateserial \
-            -CA "${root_ca_cert}" \
-            -CAkey "${root_ca_private_key}" \
-            -in "${certs_dir}/request.csr" \
-            -out "${certs_dir}/cert.pem"
-    else
-        openssl x509 -req -days 365 -sha256 \
-            -CAcreateserial \
-            -CA "${ROOT_CA_DIR}/cert.pem" \
-            -CAkey "${ROOT_CA_DIR}/private.key" \
-            -extfile "${extensions_file}" \
-            -extensions "${extensions_name}" \
-            -in "${certs_dir}/request.csr" \
-            -out "${certs_dir}/cert.pem"
-    fi
-
-    # Convert certificate to PKCS12 format
-    openssl pkcs12 -export \
-        -name "${cert_name}" \
-        -certfile "${root_ca_cert}" \
-        -in "${certs_dir}/cert.pem" \
-        -inkey "${certs_dir}/private.key" \
-        -passout "pass:${PKCS12_PASSWORD}" \
-        -out "${certs_dir}/cert.p12"
-}
-
-# generate_ipa_pkcs12_certificates $ipa_fqdn $ipa_domain
-function generate_ipa_pkcs12_certificates {
-
-    host=$1
-    if [ -z "$host" ]; then
-        echo "ERROR: ipa-host-fqdn is not set"
-        echo
-        echo "usage: $0 create ipa-host-fqdn domain"
-        exit 0;
-    fi
-
-    domain=$2
-    if [ -z "$domain" ]; then
+    domain_name=$1
+    if [ -z "${domain_name}" ]; then
         echo "ERROR: domain is not set"
         echo
-        echo "usage: $0 create ipa-host-fqdn domain"
+        echo "usage: $0 ca <domain>"
         exit 0;
     fi
+    realm=${domain_name^^}
 
-    # Generate certificates folder structure
+    export REALM_NAME=${realm}
+
+    # Create certificates folder structure
     mkdir -p "${ROOT_CA_DIR}"
-    mkdir -p "${DIRSRV_CERTS_DIR}/$host"
-    mkdir -p "${HTTPD_CERTS_DIR}/$host"
-    mkdir -p "${PKINIT_CERTS_DIR}/$host"
 
-    # Generate root CA
+    # Create root CA
     if [ ! -f "${ROOT_CA_DIR}/private.key" ]; then
-        openssl genrsa \
-                -out "${ROOT_CA_DIR}/private.key" 4096
+        # create aes encrypted private key
+        openssl genrsa -out "${ROOT_CA_DIR}/private.key" 4096
 
-        openssl req -new -x509 -sha256 -nodes -days 3650 \
-                -subj "/C=US/ST=Test/L=Testing/O=Default" \
+        # create certificate, 1826 days = 5 years
+        openssl req -x509 -new -nodes -sha256 -days 1826 \
+                -subj "/C=US/ST=Test/L=Testing/O=Default/CN=Test Root CA" \
                 -key "${ROOT_CA_DIR}/private.key" \
                 -out "${ROOT_CA_DIR}/cert.pem"
     fi
-
-    # Generate a certificate for the Directory Server
-    if [ ! -f "${DIRSRV_CERTS_DIR}/$host/cert.pem" ]; then
-        generate_ipa_pkcs12_certificate \
-            "dirsrv-cert" \
-            "$host" \
-            "${DIRSRV_CERTS_DIR}/$host" \
-            "${ROOT_CA_DIR}/cert.pem" \
-            "${ROOT_CA_DIR}/private.key"
-    fi
-
-    # Generate a certificate for the Apache server
-    if [ ! -f "${HTTPD_CERTS_DIR}/$host/cert.pem" ]; then
-        generate_ipa_pkcs12_certificate \
-            "httpd-cert" \
-            "$host" \
-            "${HTTPD_CERTS_DIR}/$host" \
-            "${ROOT_CA_DIR}/cert.pem" \
-            "${ROOT_CA_DIR}/private.key"
-    fi
-
-    # Generate a certificate for the KDC PKINIT
-    if [ ! -f "${PKINIT_CERTS_DIR}/$host/cert.pem" ]; then
-        export REALM=${domain^^}
-
-        generate_ipa_pkcs12_certificate \
-            "pkinit-cert" \
-            "$host" \
-            "${PKINIT_CERTS_DIR}/$host" \
-            "${ROOT_CA_DIR}/cert.pem" \
-            "${ROOT_CA_DIR}/private.key" \
-            "${PKINIT_CERTS_DIR}/extensions.conf" \
-            "kdc_cert"
-    fi
 }
 
-# delete_ipa_pkcs12_certificates $ipa_fqdn
-function delete_ipa_pkcs12_certificates {
+# create_host_pkcs12_certificate \
+#    $cert_name $certs_dir $root_ca_cert $extensions_file
+function create_host_pkcs12_certificate {
 
-    host=$1
-    if [ -z "$host" ]; then
-        echo "ERROR: ipa-host-fqdn is not set"
+    cert_name=$1
+    certs_dir=$2
+    root_ca_cert=$3
+    extensions_file=$4
+
+    # Create CSR and private key
+    openssl req -new -nodes -newkey rsa:4096 \
+                -subj "/C=US/ST=Test/L=Testing/O=Default/CN=${cert_name}" \
+                -keyout "${certs_dir}/private.key" \
+                -out "${certs_dir}/request.csr"
+
+    # Sign CSR to create PEM certificate
+    openssl x509 -req -days 1460 -sha256 -CAcreateserial \
+                -CAkey "${ROOT_CA_DIR}/private.key" \
+                -CA "${root_ca_cert}" \
+                -in "${certs_dir}/request.csr" \
+                -out "${certs_dir}/cert.pem" \
+                -extfile "${extensions_file}"
+
+    # Convert certificate to PKCS12 format
+    openssl pkcs12 -export \
+            -name "${cert_name}" \
+            -certfile "${root_ca_cert}" \
+            -passout "pass:${PKCS12_PASSWORD}" \
+            -inkey "${certs_dir}/private.key" \
+            -in "${certs_dir}/cert.pem" \
+            -out "${certs_dir}/cert.p12"
+}
+
+# create_ipa_pkcs12_certificates \
+#    $host_fqdn $domain_name
+function create_host_certificates {
+
+    host_fqdn=$1
+    if [ -z "${host_fqdn}" ]; then
+        echo "ERROR: host-fqdn is not set"
         echo
-        echo "usage: $0 delete ipa-host-fqdn"
+        echo "usage: $0 create <host-fqdn> [<domain>]"
         exit 0;
     fi
 
-    rm -f certificates/*/"$host"/*
-    rm -f "${ROOT_CA_DIR}"/*
+    domain_name=$2
+    [ -z "${domain_name}" ] && domain_name=${host_fqdn#*.*}
+    if [ -z "${domain_name}" ]; then
+        echo "ERROR: domain is not set and can not be created from host fqdn"
+        echo
+        echo "usage: $0 create <host-fqdn> [<domain>]"
+        exit 0;
+    fi
+    realm=${domain_name^^}
+
+    export HOST_FQDN=${host_fqdn}
+    export REALM_NAME=${realm}
+
+    if [ ! -f "${ROOT_CA_DIR}/private.key" ]; then
+        create_ca "${domain_name}"
+    fi
+
+    # Create certificates folder structure
+    mkdir -p "${DIRSRV_CERTS_DIR}/${host_fqdn}"
+    mkdir -p "${HTTPD_CERTS_DIR}/${host_fqdn}"
+    mkdir -p "${PKINIT_CERTS_DIR}/${host_fqdn}"
+
+    # Create a certificate for the Directory Server
+    if [ ! -f "${DIRSRV_CERTS_DIR}/${host_fqdn}/cert.pem" ]; then
+        create_host_pkcs12_certificate \
+            "dirsrv-cert" \
+            "${DIRSRV_CERTS_DIR}/${host_fqdn}" \
+            "${ROOT_CA_DIR}/cert.pem" \
+            "${EXTENSIONS_CONF}"
+    fi
+
+    # Create a certificate for the Apache server
+    if [ ! -f "${HTTPD_CERTS_DIR}/${host_fqdn}/cert.pem" ]; then
+        create_host_pkcs12_certificate \
+            "httpd-cert" \
+            "${HTTPD_CERTS_DIR}/${host_fqdn}" \
+            "${ROOT_CA_DIR}/cert.pem" \
+            "${EXTENSIONS_CONF}"
+    fi
+
+    # Create a certificate for the KDC PKINIT
+    if [ ! -f "${PKINIT_CERTS_DIR}/${host_fqdn}/cert.pem" ]; then
+        create_host_pkcs12_certificate \
+            "pkinit-cert" \
+            "${PKINIT_CERTS_DIR}/${host_fqdn}" \
+            "${ROOT_CA_DIR}/cert.pem" \
+            "${PKINIT_EXTENSIONS_CONF}"
+    fi
+}
+
+# delete_host_certificates \
+#     $host_fqdn
+function delete_host_certificates {
+
+    host_fqdn=$1
+    if [ -z "${host_fqdn}" ]; then
+        echo "ERROR: host-fqdn is not set"
+        echo
+        echo "usage: $0 delete <host-fqdn>"
+        exit 0;
+    fi
+
+    rm -rf certificates/*/"${host_fqdn}"/
+}
+
+# cleanup \
+#     $host_fqdn
+function cleanup {
+
+    rm -rf certificates/*/
 }
 
 # Entrypoint
 case "$1" in
+  ca)
+    create_ca "$2"
+    ;;
   create)
-    generate_ipa_pkcs12_certificates "$2" "$3"
+    create_host_certificates "$2" "$3"
     ;;
   delete)
-    delete_ipa_pkcs12_certificates "$2"
+    delete_host_certificates "$2"
+    ;;
+  cleanup)
+    cleanup
     ;;
   *)
-    echo $"Usage: $0 {create|delete}"
+    echo $"Usage: $0 {create|delete|ca|cleanup}"
     ;;
 esac

utils/build-galaxy-release.sh

@@ -8,18 +8,24 @@ pwd=$(pwd)
 
 usage() {
     cat <<EOF
-Usage: $prog [options] [namespace] [collection]
+Usage: $prog [options] [<namespace> <name>]
 
 Build Anible Collection for ansible-freeipa.
 
-The namespace defaults to freeipa an collection defaults to ansible_freeipa
-if namespace and collection are not given. Namespace and collection can not
-be givedn without the other one.
+The namespace defaults to freeipa an name defaults to ansible_freeipa,
+if namespace and name are not given. Namespace and name need to be set
+together.
 
 Options:
   -a          Add all files, no only files known to git repo
   -k          Keep build directory
   -i          Install the generated collection
+  -o <A.B.C>  Build offline without using git, using version A.B.C
+              Also enables -a
+  -p <path>   Installation the generated collection in the path, the
+              ansible_collections sub directory will be created and will
+              contain the collection: ansible_collections/<namespace>/<name>
+              Also enables -i
   -h          Print this help
 
 EOF
@@ -28,7 +34,10 @@ EOF
 all=0
 keep=0
 install=0
-while getopts "ahki" arg; do
+path=
+offline=
+galaxy_version=
+while getopts "ahkio:p:" arg; do
     case $arg in
         a)
             all=1
@@ -43,6 +52,15 @@ while getopts "ahki" arg; do
         i)
             install=1
             ;;
+        o)
+            galaxy_version=$OPTARG
+            offline=1
+            all=1
+            ;;
+        p)
+            path=$OPTARG
+            install=1
+            ;;
         \?)
             echo
             usage
@@ -57,25 +75,26 @@ if [ $# != 0 ] && [ $# != 2 ]; then
     exit 1
 fi
 namespace="${1-freeipa}"
-collection="${2-ansible_freeipa}"
+name="${2-ansible_freeipa}"
 if [ -z "$namespace" ]; then
     echo "Namespace might not be empty"
     exit 1
 fi
-if [ -z "$collection" ]; then
-    echo "Collection might not be empty"
+if [ -z "$name" ]; then
+    echo "Name might not be empty"
     exit 1
 fi
-collection_prefix="${namespace}.${collection}"
+collection_prefix="${namespace}.${name}"
 
-galaxy_version=$(git describe --tags 2>/dev/null | sed -e "s/^v//")
+[ -z "$galaxy_version" ] && \
+    galaxy_version=$(git describe --tags 2>/dev/null | sed -e "s/^v//")
 
 if [ -z "$galaxy_version" ]; then
     echo "Version could not be detected"
     exit 1
 fi
 
-echo "Building collection: ${namespace}-${collection}-${galaxy_version}"
+echo "Building collection: ${namespace}-${name}-${galaxy_version}"
 
 GALAXY_BUILD=".galaxy-build"
 
@@ -103,14 +122,19 @@ cd "$GALAXY_BUILD" || exit 1
 
 sed -i -e "s/version: .*/version: \"$galaxy_version\"/" galaxy.yml
 sed -i -e "s/namespace: .*/namespace: \"$namespace\"/" galaxy.yml
-sed -i -e "s/name: .*/name: \"$collection\"/" galaxy.yml
+sed -i -e "s/name: .*/name: \"$name\"/" galaxy.yml
 
 find . -name "*~" -exec rm {} \;
+find . -name "__py*__" -exec rm -rf {} \;
 
 
-echo "Creating CHANGELOG.rst..."
-"$(dirname "$0")/changelog" --galaxy > CHANGELOG.rst
-echo -e "\033[ACreating CHANGELOG.rst... \033[32;1mDONE\033[0m"
+if [ "$offline" != "" ]; then
+    echo "Creating CHANGELOG.rst..."
+    "$(dirname "$0")/changelog" --galaxy > CHANGELOG.rst
+    echo -e "\033[ACreating CHANGELOG.rst... \033[32;1mDONE\033[0m"
+else
+    echo "Empty changelog, offline generated." > CHANGELOG.rst
+fi
 
 sed -i -e "s/ansible.module_utils.ansible_freeipa_module/ansible_collections.${collection_prefix}.plugins.module_utils.ansible_freeipa_module/" plugins/modules/*.py
 
@@ -132,6 +156,13 @@ python utils/create_action_group.py "meta/runtime.yml" "$collection_prefix"
 #    ln -sf ../../roles/*/action_plugins/*.py .
 #})
 
+# Adapt inventory plugin and inventory plugin README
+echo "Fixing inventory plugin and doc..."
+sed -i -e "s/plugin: freeipa/plugin: ${collection_prefix}.freeipa/g" plugins/inventory/freeipa.py
+sed -i -e "s/choices: \[\"freeipa\"\]/choices: \[\"${collection_prefix}.freeipa\"\]/g" plugins/inventory/freeipa.py
+sed -i -e "s/plugin: freeipa/plugin: ${collection_prefix}.freeipa/g" README-inventory-plugin-freeipa.md
+echo -e "\033[AFixing inventory plugin and doc... \033[32;1mDONE\033[0m"
+
 for doc_fragment in plugins/doc_fragments/*.py; do
     fragment=$(basename -s .py "$doc_fragment")
 
@@ -198,6 +229,6 @@ else
 fi
 
 if [ $install == 1 ]; then
-    echo "Installing collection ${namespace}-${collection}-${galaxy_version}.tar.gz ..."
-    ansible-galaxy collection install "${namespace}-${collection}-${galaxy_version}.tar.gz" --force
+    echo "Installing collection ${namespace}-${name}-${galaxy_version}.tar.gz ..."
+    ansible-galaxy collection install ${path:+"-p$path"} "${namespace}-${name}-${galaxy_version}.tar.gz" --force ${offline/1/--offline}
 fi