fix(inventory): Fix inventory source caching

Fix inventory source caching by separating the fetching of objects and
populating the inventory. This way objects can be fetched from the K8S
API or from a configured cached and the cache related parameters on the
plugin now actually work.

The inventory source cache was tested with the ansible.builtin.jsonfile
cache plugin and 100k hosts (~2G JSON file). Though it took a noticeable
amount of time for the inventory plugin to run it worked fine and no
failures could be observed.

Signed-off-by: Felix Matouschek <fmatouschek@redhat.com>
This commit is contained in:
Felix Matouschek
2024-07-08 17:25:28 +02:00
parent a228fe2c26
commit 060ac97b68
13 changed files with 408 additions and 218 deletions

View File

@@ -10,9 +10,12 @@ ansible-inventory -i test.kubevirt.yml -y --list --output empty.yml "$@"
ansible-playbook playbook.yml "$@"
ansible-inventory -i test.kubevirt.yml -y --list --output all.yml "$@"
ansible-inventory -i test.cache.kubevirt.yml -y --list --output cache_before.yml "$@"
ansible-inventory -i test.label.kubevirt.yml -y --list --output label.yml "$@"
ansible-inventory -i test.net.kubevirt.yml -y --list --output net.yml "$@"
ansible-playbook cleanup.yml "$@"
ansible-inventory -i test.cache.kubevirt.yml -y --list --output cache_after.yml "$@"
ansible-playbook verify.yml "$@"

View File

@@ -0,0 +1,9 @@
---
plugin: kubevirt.core.kubevirt
name: test
namespaces:
- default
create_groups: true
cache: true
cache_plugin: ansible.builtin.jsonfile
cache_connection: kubevirt-cache

View File

@@ -38,3 +38,16 @@
ansible.builtin.assert:
that:
- inv_net['all']['children']['label_app_test']['hosts'] | length == 1
- name: Read cached inventory after VM creation
ansible.builtin.include_vars:
file: cache_before.yml
name: inv_cache_before
- name: Read cached inventory after VM deletion
ansible.builtin.include_vars:
file: cache_after.yml
name: inv_cache_after
- name: Assert cached inventories are populated
ansible.builtin.assert:
that:
- inv_cache_before == inv_all
- inv_cache_before == inv_cache_after

View File

@@ -8,7 +8,6 @@ __metaclass__ = type
import pytest
from ansible_collections.kubevirt.core.plugins.inventory.kubevirt import (
InventoryOptions,
)
@@ -60,19 +59,25 @@ WINDOWS_VMI_4 = merge_dicts(
@pytest.mark.parametrize(
"client,vmi,expected",
"vmi,expected",
[
({"vmis": [BASE_VMI]}, BASE_VMI, False),
({"vmis": [WINDOWS_VMI_1]}, WINDOWS_VMI_1, True),
({"vmis": [WINDOWS_VMI_2]}, WINDOWS_VMI_2, True),
({"vmis": [WINDOWS_VMI_3]}, WINDOWS_VMI_3, True),
({"vmis": [WINDOWS_VMI_4]}, WINDOWS_VMI_4, True),
(BASE_VMI, False),
(WINDOWS_VMI_1, True),
(WINDOWS_VMI_2, True),
(WINDOWS_VMI_3, True),
(WINDOWS_VMI_4, True),
],
indirect=["client"],
)
def test_ansible_connection_winrm(inventory, hosts, client, vmi, expected):
inventory.populate_inventory_from_namespace(
client, "", DEFAULT_NAMESPACE, InventoryOptions()
def test_ansible_connection_winrm(inventory, hosts, vmi, expected):
inventory.populate_inventory(
{
"default_hostname": "test",
"cluster_domain": "test.com",
"namespaces": {
"default": {"vms": [], "vmis": [vmi], "services": {}},
},
},
InventoryOptions(),
)
host = f"{DEFAULT_NAMESPACE}-{vmi['metadata']['name']}"

View File

@@ -6,8 +6,6 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import pytest
from ansible_collections.kubevirt.core.plugins.inventory.kubevirt import (
InventoryOptions,
)
@@ -36,16 +34,10 @@ VMI = {
}
@pytest.mark.parametrize(
"client",
[{"vmis": [VMI]}],
indirect=["client"],
)
def test_set_composable_vars(
inventory,
groups,
hosts,
client,
):
inventory._options = {
"compose": {"set_from_another_var": "vmi_node_name"},
@@ -53,8 +45,15 @@ def test_set_composable_vars(
"keyed_groups": [{"prefix": "fedora", "key": "vmi_guest_os_info.versionId"}],
"strict": True,
}
inventory.populate_inventory_from_namespace(
client, "", DEFAULT_NAMESPACE, InventoryOptions()
inventory.populate_inventory(
{
"default_hostname": "test",
"cluster_domain": "test.com",
"namespaces": {
"default": {"vms": [], "vmis": [VMI], "services": {}},
},
},
InventoryOptions(),
)
host = f"{DEFAULT_NAMESPACE}-testvmi"

View File

@@ -6,17 +6,10 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import pytest
from ansible_collections.kubevirt.core.plugins.inventory.kubevirt import (
InventoryOptions,
)
from ansible_collections.kubevirt.core.tests.unit.plugins.inventory.constants import (
DEFAULT_NAMESPACE,
)
VM1 = {
"metadata": {
"name": "testvm1",
@@ -50,16 +43,16 @@ VMI1 = {
}
@pytest.mark.parametrize(
"client",
[
({"vms": [VM1, VM2], "vmis": [VMI1]}),
],
indirect=["client"],
)
def test_stopped_vm(inventory, hosts, client):
inventory.populate_inventory_from_namespace(
client, "", DEFAULT_NAMESPACE, InventoryOptions()
def test_stopped_vm(inventory, hosts):
inventory.populate_inventory(
{
"default_hostname": "test",
"cluster_domain": "test.com",
"namespaces": {
"default": {"vms": [VM1, VM2], "vmis": [VMI1], "services": {}},
},
},
InventoryOptions(),
)
# The running VM should be present with ansible_host or ansible_port

View File

@@ -10,6 +10,7 @@ import pytest
from ansible_collections.kubevirt.core.plugins.inventory.kubevirt import (
InventoryModule,
InventoryOptions,
)
from ansible_collections.kubevirt.core.tests.unit.plugins.inventory.constants import (
@@ -257,6 +258,56 @@ def test_get_cluster_domain(inventory, client):
assert inventory.get_cluster_domain(client) == DEFAULT_BASE_DOMAIN
@pytest.mark.parametrize(
"results,expected",
[
(
{
"cluster_domain": "example.com",
"default_hostname": "test",
"namespaces": {},
},
0,
),
(
{
"cluster_domain": "example.com",
"default_hostname": "test",
"namespaces": {"test": {"vms": [], "vmis": [], "services": {}}},
},
1,
),
(
{
"cluster_domain": "example.com",
"default_hostname": "test",
"namespaces": {
"test": {"vms": [], "vmis": [], "services": {}},
"test2": {"vms": [], "vmis": [], "services": {}},
},
},
2,
),
],
)
def test_populate_inventory(mocker, inventory, results, expected):
populate_inventory_from_namespace = mocker.patch.object(
inventory, "populate_inventory_from_namespace"
)
inventory.populate_inventory(results, InventoryOptions())
opts = InventoryOptions(
base_domain=results["cluster_domain"], name=results["default_hostname"]
)
calls = [
mocker.call(namespace, data, opts)
for namespace, data in results["namespaces"].items()
]
populate_inventory_from_namespace.assert_has_calls(calls)
assert len(calls) == expected
@pytest.mark.parametrize(
"labels,expected",
[

View File

@@ -17,127 +17,93 @@ from ansible_collections.kubevirt.core.tests.unit.plugins.inventory.constants im
DEFAULT_NAMESPACE,
)
from ansible_collections.kubevirt.core.plugins.inventory import kubevirt
@pytest.mark.parametrize(
"config_data,expected",
"opts,namespaces",
[
(
None,
{
"name": "default-hostname",
"namespaces": [DEFAULT_NAMESPACE],
"opts": InventoryOptions(),
},
InventoryOptions(),
[DEFAULT_NAMESPACE],
),
(
{
"name": "test",
},
{
"name": "test",
"namespaces": [DEFAULT_NAMESPACE],
"opts": InventoryOptions(),
},
InventoryOptions(namespaces=["test"]),
["test"],
),
(
{"name": "test", "namespaces": ["test"]},
{"name": "test", "namespaces": ["test"], "opts": InventoryOptions()},
),
(
{
"name": "test",
"namespaces": ["test"],
"use_service": True,
"create_groups": True,
"append_base_domain": True,
"base_domain": "test-domain",
},
{
"name": "test",
"namespaces": ["test"],
"opts": InventoryOptions(
use_service=True,
create_groups=True,
append_base_domain=True,
base_domain="test-domain",
),
},
),
(
{
"name": "test",
"namespaces": ["test"],
"use_service": True,
"create_groups": True,
"append_base_domain": True,
"base_domain": "test-domain",
"network_name": "test-network",
},
{
"name": "test",
"namespaces": ["test"],
"opts": InventoryOptions(
use_service=True,
create_groups=True,
append_base_domain=True,
base_domain="test-domain",
network_name="test-network",
),
},
),
(
{
"name": "test",
"namespaces": ["test"],
"use_service": True,
"create_groups": True,
"append_base_domain": True,
"base_domain": "test-domain",
"interface_name": "test-interface",
},
{
"name": "test",
"namespaces": ["test"],
"opts": InventoryOptions(
use_service=True,
create_groups=True,
append_base_domain=True,
base_domain="test-domain",
network_name="test-interface",
),
},
InventoryOptions(namespaces=["test1", "test2"]),
["test1", "test2"],
),
],
)
def test_fetch_objects(mocker, inventory, config_data, expected):
mocker.patch.object(kubevirt, "get_api_client")
mocker.patch.object(
inventory, "get_default_host_name", return_value="default-hostname"
)
cluster_domain = "test.com"
mocker.patch.object(inventory, "get_cluster_domain", return_value=cluster_domain)
expected["opts"].base_domain = expected["opts"].base_domain or cluster_domain
def test_fetch_objects(mocker, inventory, opts, namespaces):
get_available_namespaces = mocker.patch.object(
inventory, "get_available_namespaces", return_value=[DEFAULT_NAMESPACE]
)
populate_inventory_from_namespace = mocker.patch.object(
inventory, "populate_inventory_from_namespace"
get_vms_for_namespace = mocker.patch.object(
inventory, "get_vms_for_namespace", return_value=[{}]
)
get_vmis_for_namespace = mocker.patch.object(
inventory, "get_vmis_for_namespace", return_value=[{}]
)
get_ssh_services_for_namespace = mocker.patch.object(
inventory, "get_ssh_services_for_namespace", return_value=[]
)
get_default_hostname = mocker.patch.object(
inventory, "get_default_hostname", return_value="default-hostname"
)
get_cluster_domain = mocker.patch.object(
inventory, "get_cluster_domain", return_value="test.com"
)
inventory.fetch_objects(config_data)
inventory.fetch_objects(mocker.Mock(), opts)
if config_data and "namespaces" in config_data:
if opts.namespaces:
get_available_namespaces.assert_not_called()
else:
get_available_namespaces.assert_called()
populate_inventory_from_namespace.assert_has_calls(
[
mocker.call(mocker.ANY, expected["name"], namespace, expected["opts"])
for namespace in expected["namespaces"]
]
get_vms_for_namespace.assert_has_calls(
[mocker.call(mocker.ANY, namespace, opts) for namespace in namespaces]
)
get_vmis_for_namespace.assert_has_calls(
[mocker.call(mocker.ANY, namespace, opts) for namespace in namespaces]
)
get_ssh_services_for_namespace.assert_has_calls(
[mocker.call(mocker.ANY, namespace) for namespace in namespaces]
)
get_default_hostname.assert_called_once()
get_cluster_domain.assert_called_once()
def test_fetch_objects_early_return(mocker, inventory):
get_available_namespaces = mocker.patch.object(
inventory, "get_available_namespaces", return_value=[DEFAULT_NAMESPACE]
)
get_vms_for_namespace = mocker.patch.object(
inventory, "get_vms_for_namespace", return_value=[]
)
get_vmis_for_namespace = mocker.patch.object(
inventory, "get_vmis_for_namespace", return_value=[]
)
get_ssh_services_for_namespace = mocker.patch.object(
inventory, "get_ssh_services_for_namespace"
)
get_default_hostname = mocker.patch.object(
inventory, "get_default_hostname", return_value="default-hostname"
)
get_cluster_domain = mocker.patch.object(
inventory, "get_cluster_domain", return_value="test.com"
)
inventory.fetch_objects(mocker.Mock(), InventoryOptions())
get_available_namespaces.assert_called_once()
get_vms_for_namespace.assert_called_once_with(
mocker.ANY, DEFAULT_NAMESPACE, InventoryOptions()
)
get_vmis_for_namespace.assert_called_once_with(
mocker.ANY, DEFAULT_NAMESPACE, InventoryOptions()
)
get_ssh_services_for_namespace.assert_not_called()
get_default_hostname.assert_called_once()
get_cluster_domain.assert_called_once()

View File

@@ -13,39 +13,151 @@ from ansible_collections.kubevirt.core.plugins.inventory import (
)
from ansible_collections.kubevirt.core.plugins.inventory.kubevirt import (
InventoryOptions,
KubeVirtInventoryException,
)
@pytest.mark.parametrize(
"cache",
"config_data,expected",
[
True,
False,
(
{},
InventoryOptions(),
),
(
{
"name": "test",
},
InventoryOptions(name="test"),
),
(
{"name": "test", "namespaces": ["test"]},
InventoryOptions(name="test", namespaces=["test"]),
),
(
{
"name": "test",
"namespaces": ["test"],
"use_service": True,
"create_groups": True,
"append_base_domain": True,
"base_domain": "test-domain",
},
InventoryOptions(
use_service=True,
create_groups=True,
append_base_domain=True,
base_domain="test-domain",
name="test",
namespaces=["test"],
),
),
(
{
"name": "test",
"namespaces": ["test"],
"use_service": True,
"create_groups": True,
"append_base_domain": True,
"base_domain": "test-domain",
"network_name": "test-network",
},
InventoryOptions(
use_service=True,
create_groups=True,
append_base_domain=True,
base_domain="test-domain",
network_name="test-network",
name="test",
namespaces=["test"],
),
),
(
{
"name": "test",
"namespaces": ["test"],
"use_service": True,
"create_groups": True,
"append_base_domain": True,
"base_domain": "test-domain",
"interface_name": "test-interface",
},
InventoryOptions(
use_service=True,
create_groups=True,
append_base_domain=True,
base_domain="test-domain",
network_name="test-interface",
name="test",
namespaces=["test"],
),
),
],
)
def test_parse(mocker, inventory, cache):
path = "/testpath"
cache_prefix = "test-prefix"
config_data = {"host_format": "test-format"}
mocker.patch.dict(inventory._cache, {cache_prefix: {"test-key": "test-value"}})
get_cache_prefix = mocker.patch.object(
inventory, "_get_cache_prefix", return_value=cache_prefix
)
def test_config_data_to_opts(mocker, inventory, config_data, expected):
read_config_data = mocker.patch.object(
inventory, "_read_config_data", return_value=config_data
)
mocker.patch.object(inventory, "get_cache_key")
mocker.patch.object(inventory, "get_option")
mocker.patch.object(kubevirt, "get_api_client")
mocker.patch.object(inventory, "fetch_objects")
populate_inventory = mocker.patch.object(inventory, "populate_inventory")
inventory.parse(None, None, "", False)
populate_inventory.assert_called_once_with(mocker.ANY, expected)
@pytest.mark.parametrize(
"cache_parse,cache_option,cache_data,expected",
[
(True, True, {"test-key": {"something": "something"}}, True),
(None, True, {"test-key": {"something": "something"}}, True),
(False, True, {"test-key": {"something": "something"}}, False),
(True, False, {"test-key": {"something": "something"}}, False),
(None, False, {"test-key": {"something": "something"}}, False),
(False, False, {"test-key": {"something": "something"}}, False),
(True, True, {"test-key2": {"something": "something"}}, False),
(None, True, {"test-key2": {"something": "something"}}, False),
],
)
def test_use_of_cache(
mocker, inventory, cache_parse, cache_option, cache_data, expected
):
path = "/testpath"
config_data = {"host_format": "test-format"}
mocker.patch.dict(inventory._cache, cache_data)
read_config_data = mocker.patch.object(
inventory, "_read_config_data", return_value=config_data
)
get_cache_key = mocker.patch.object(
inventory, "get_cache_key", return_value="test-key"
)
get_option = mocker.patch.object(inventory, "get_option", return_value=cache_option)
get_api_client = mocker.patch.object(kubevirt, "get_api_client")
fetch_objects = mocker.patch.object(inventory, "fetch_objects")
populate_inventory = mocker.patch.object(inventory, "populate_inventory")
inventory.parse(None, None, path, cache)
if cache_parse is None:
inventory.parse(None, None, path)
else:
inventory.parse(None, None, path, cache_parse)
get_cache_prefix.assert_called_once_with(path)
opts = InventoryOptions(config_data=config_data)
get_cache_key.assert_called_once_with(path)
get_option.assert_called_once_with("cache")
read_config_data.assert_called_once_with(path)
if cache:
if expected:
get_api_client.assert_not_called()
fetch_objects.assert_not_called()
else:
fetch_objects.assert_called_once_with(config_data)
get_api_client.assert_called_once_with(**config_data)
fetch_objects.assert_called_once_with(mocker.ANY, opts)
populate_inventory.assert_called_once_with(mocker.ANY, opts)
@pytest.mark.parametrize(
@@ -57,8 +169,10 @@ def test_parse(mocker, inventory, cache):
)
def test_k8s_client_missing(mocker, inventory, present):
mocker.patch.object(kubevirt, "HAS_K8S_MODULE_HELPER", present)
mocker.patch.object(inventory, "_get_cache_prefix")
mocker.patch.object(inventory, "_read_config_data")
mocker.patch.object(kubevirt, "get_api_client")
mocker.patch.object(inventory, "_read_config_data", return_value={})
mocker.patch.object(inventory, "get_cache_key")
mocker.patch.object(inventory, "get_option")
fetch_objects = mocker.patch.object(inventory, "fetch_objects")
if present:

View File

@@ -83,7 +83,7 @@ def test_populate_inventory_from_namespace(
):
_vms = {vm["metadata"]["name"]: vm for vm in vms}
_vmis = {vmi["metadata"]["name"]: vmi for vmi in vmis}
opts = InventoryOptions()
opts = InventoryOptions(name="test")
def format_hostname(obj):
return opts.host_format.format(
@@ -99,6 +99,7 @@ def test_populate_inventory_from_namespace(
f"namespace_{DEFAULT_NAMESPACE}",
)
obj_is_valid_calls = []
add_host_side_effects = []
add_host_calls = []
set_vars_from_vm_calls = []
@@ -108,33 +109,27 @@ def test_populate_inventory_from_namespace(
# For each VM add the expected calls
# Also add expected calls for VMIs for which a VM exists
for name, vm in _vms.items():
obj_is_valid_calls.append(mocker.call(vm))
hostname = format_hostname(vm)
add_host_side_effects.append(hostname)
add_host_calls.append(add_host_call(vm))
set_vars_from_vm_calls.append(mocker.call(hostname, vm, opts))
if name in _vmis.keys():
set_vars_from_vmi_calls.append(mocker.call(hostname, _vmis[name], [], opts))
set_vars_from_vmi_calls.append(mocker.call(hostname, _vmis[name], {}, opts))
set_composable_vars_calls.append(mocker.call(hostname))
# For each VMI add the expected calls
# Do not add for VMIs for which a VM exists
for name, vmi in _vmis.items():
obj_is_valid_calls.append(mocker.call(vmi))
if name not in _vms.keys():
hostname = format_hostname(vmi)
add_host_side_effects.append(hostname)
add_host_calls.append(add_host_call(vmi))
set_vars_from_vmi_calls.append(mocker.call(hostname, vmi, [], opts))
set_vars_from_vmi_calls.append(mocker.call(hostname, vmi, {}, opts))
set_composable_vars_calls.append(mocker.call(hostname))
get_vms_for_namespace = mocker.patch.object(
inventory, "get_vms_for_namespace", return_value=_vms.values()
)
get_vmis_for_namespace = mocker.patch.object(
inventory, "get_vmis_for_namespace", return_value=_vmis.values()
)
get_ssh_services_for_namespace = mocker.patch.object(
inventory, "get_ssh_services_for_namespace", return_value=[]
)
obj_is_valid = mocker.patch.object(inventory, "obj_is_valid", return_value=True)
add_host = mocker.patch.object(
inventory, "add_host", side_effect=add_host_side_effects
)
@@ -142,23 +137,20 @@ def test_populate_inventory_from_namespace(
set_vars_from_vmi = mocker.patch.object(inventory, "set_vars_from_vmi")
set_composable_vars = mocker.patch.object(inventory, "set_composable_vars")
inventory.populate_inventory_from_namespace(None, "test", DEFAULT_NAMESPACE, opts)
# These should always get called once
get_vms_for_namespace.assert_called_once_with(None, DEFAULT_NAMESPACE, opts)
get_vmis_for_namespace.assert_called_once_with(None, DEFAULT_NAMESPACE, opts)
inventory.populate_inventory_from_namespace(
DEFAULT_NAMESPACE, {"vms": vms, "vmis": vmis, "services": {}}, opts
)
# Assert it tries to add the expected vars for all provided VMs/VMIs
obj_is_valid.assert_has_calls(obj_is_valid_calls)
set_vars_from_vm.assert_has_calls(set_vars_from_vm_calls)
set_vars_from_vmi.assert_has_calls(set_vars_from_vmi_calls)
set_composable_vars.assert_has_calls(set_composable_vars_calls)
# If no VMs or VMIs were provided the function should not add any groups
if vms or vmis:
get_ssh_services_for_namespace.assert_called_once_with(None, DEFAULT_NAMESPACE)
assert list(groups.keys()) == ["test", f"namespace_{DEFAULT_NAMESPACE}"]
else:
get_ssh_services_for_namespace.assert_not_called()
assert not list(groups.keys())
# Assert the expected amount of hosts was added