Fix inventory cache interface (#50446)

* Replace InventoryFileCacheModule with a better developer-interface

Use new interface for inventory plugins with backwards compatibility

Auto-update the backing cache-plugin if the cache has changed after parsing the inventory plugin

* Update CacheModules to use the config system and add a deprecation warning if they are being imported directly rather than using cache_loader

* Fix foreman inventory caching

* Add tests

* Add integration test to check that fact caching works normally with cache plugins using ansible.constants and inventory caching provides a helpful error for non-compatible cache plugins

* Add some developer documentation for inventory and cache plugins

* Add user documentation for inventory caching

* Add deprecation docs

* Apply suggestions from docs review

* Add changelog

test/integration/targets/inventory_foreman/inspect_cache.yml

@@ -0,0 +1,31 @@
+---
+- hosts: localhost
+  vars:
+    foreman_stub_host: "{{ lookup('env', 'FOREMAN_HOST') }}"
+    foreman_stub_port: "{{ lookup('env', 'FOREMAN_PORT') }}"
+    foreman_stub_api_path: /api/v2
+    cached_hosts_key: "http://{{ foreman_stub_host }}:{{ foreman_stub_port }}{{ foreman_stub_api_path }}/hosts"
+  tasks:
+    - name: verify a cache file was created
+      find:
+        path:
+          - ./foreman_cache
+      register: matching_files
+
+    - assert:
+        that:
+          - matching_files.matched == 1
+    - name: read the cached inventory
+      set_fact:
+        contents: "{{ lookup('file', matching_files.files.0.path) }}"
+
+    - name: extract all the host names
+      set_fact:
+        cached_hosts: "{{ contents[cached_hosts_key] | json_query('[*].name') }}"
+
+    - assert:
+        that:
+          "'{{ item }}' in cached_hosts"
+      loop:
+        - "v6.example-780.com"
+        - "c4.j1.y5.example-487.com"

test/integration/targets/inventory_foreman/runme.sh

@@ -9,6 +9,11 @@ export FOREMAN_HOST="${FOREMAN_HOST:-localhost}"
 export FOREMAN_PORT="${FOREMAN_PORT:-8080}"
 FOREMAN_CONFIG=test-config.foreman.yaml
 
+# Set inventory caching environment variables to populate a jsonfile cache
+export ANSIBLE_INVENTORY_CACHE=True
+export ANSIBLE_INVENTORY_CACHE_PLUGIN=jsonfile
+export ANSIBLE_INVENTORY_CACHE_CONNECTION=./foreman_cache
+
 # flag for checking whether cleanup has already fired
 _is_clean=
 
@@ -33,3 +38,7 @@ validate_certs: False
 FOREMAN_YAML
 
 ansible-playbook test_foreman_inventory.yml --connection=local "$@"
+ansible-playbook inspect_cache.yml --connection=local "$@"
+
+# remove inventory cache
+rm -r ./foreman_cache

test/integration/targets/old_style_cache_plugins/aliases

@@ -0,0 +1,2 @@
+shippable/posix/group3
+skip/osx

test/integration/targets/old_style_cache_plugins/inventory_config

@@ -0,0 +1 @@
+# inventory config file for consistent source

test/integration/targets/old_style_cache_plugins/plugins/cache/redis.py

@@ -0,0 +1,141 @@
+# (c) 2014, Brian Coca, Josh Drake, et al
+# (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    cache: redis
+    short_description: Use Redis DB for cache
+    description:
+        - This cache uses JSON formatted, per host records saved in Redis.
+    version_added: "1.9"
+    requirements:
+      - redis>=2.4.5 (python lib)
+    options:
+      _uri:
+        description:
+          - A colon separated string of connection information for Redis.
+        required: True
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_CONNECTION
+        ini:
+          - key: fact_caching_connection
+            section: defaults
+      _prefix:
+        description: User defined prefix to use when creating the DB entries
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_PREFIX
+        ini:
+          - key: fact_caching_prefix
+            section: defaults
+      _timeout:
+        default: 86400
+        description: Expiration timeout for the cache plugin data
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT
+        ini:
+          - key: fact_caching_timeout
+            section: defaults
+        type: integer
+'''
+
+import time
+import json
+
+from ansible import constants as C
+from ansible.errors import AnsibleError
+from ansible.plugins.cache import BaseCacheModule
+
+try:
+    from redis import StrictRedis, VERSION
+except ImportError:
+    raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'")
+
+
+class CacheModule(BaseCacheModule):
+    """
+    A caching module backed by redis.
+    Keys are maintained in a zset with their score being the timestamp
+    when they are inserted. This allows for the usage of 'zremrangebyscore'
+    to expire keys. This mechanism is used or a pattern matched 'scan' for
+    performance.
+    """
+    def __init__(self, *args, **kwargs):
+        if C.CACHE_PLUGIN_CONNECTION:
+            connection = C.CACHE_PLUGIN_CONNECTION.split(':')
+        else:
+            connection = []
+
+        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
+        self._prefix = C.CACHE_PLUGIN_PREFIX
+        self._cache = {}
+        self._db = StrictRedis(*connection)
+        self._keys_set = 'ansible_cache_keys'
+
+    def _make_key(self, key):
+        return self._prefix + key
+
+    def get(self, key):
+
+        if key not in self._cache:
+            value = self._db.get(self._make_key(key))
+            # guard against the key not being removed from the zset;
+            # this could happen in cases where the timeout value is changed
+            # between invocations
+            if value is None:
+                self.delete(key)
+                raise KeyError
+            self._cache[key] = json.loads(value)
+
+        return self._cache.get(key)
+
+    def set(self, key, value):
+
+        value2 = json.dumps(value)
+        if self._timeout > 0:  # a timeout of 0 is handled as meaning 'never expire'
+            self._db.setex(self._make_key(key), int(self._timeout), value2)
+        else:
+            self._db.set(self._make_key(key), value2)
+
+        if VERSION[0] == 2:
+            self._db.zadd(self._keys_set, time.time(), key)
+        else:
+            self._db.zadd(self._keys_set, {key: time.time()})
+        self._cache[key] = value
+
+    def _expire_keys(self):
+        if self._timeout > 0:
+            expiry_age = time.time() - self._timeout
+            self._db.zremrangebyscore(self._keys_set, 0, expiry_age)
+
+    def keys(self):
+        self._expire_keys()
+        return self._db.zrange(self._keys_set, 0, -1)
+
+    def contains(self, key):
+        self._expire_keys()
+        return (self._db.zrank(self._keys_set, key) is not None)
+
+    def delete(self, key):
+        if key in self._cache:
+            del self._cache[key]
+        self._db.delete(self._make_key(key))
+        self._db.zrem(self._keys_set, key)
+
+    def flush(self):
+        for key in self.keys():
+            self.delete(key)
+
+    def copy(self):
+        # TODO: there is probably a better way to do this in redis
+        ret = dict()
+        for key in self.keys():
+            ret[key] = self.get(key)
+        return ret
+
+    def __getstate__(self):
+        return dict()
+
+    def __setstate__(self, data):
+        self.__init__()

test/integration/targets/old_style_cache_plugins/plugins/inventory/test.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    name: test
+    plugin_type: inventory
+    short_description: test inventory source
+    extends_documentation_fragment:
+        - inventory_cache
+'''
+
+from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
+
+
+class InventoryModule(BaseInventoryPlugin, Cacheable):
+
+    NAME = 'test'
+
+    def populate(self, hosts):
+        for host in list(hosts.keys()):
+            self.inventory.add_host(host, group='all')
+            for hostvar, hostval in hosts[host].items():
+                self.inventory.set_variable(host, hostvar, hostval)
+
+    def get_hosts(self):
+        return {'host1': {'one': 'two'}, 'host2': {'three': 'four'}}
+
+    def parse(self, inventory, loader, path, cache=True):
+        super(InventoryModule, self).parse(inventory, loader, path)
+
+        self.load_cache_plugin()
+
+        cache_key = self.get_cache_key(path)
+
+        # cache may be True or False at this point to indicate if the inventory is being refreshed
+        # get the user's cache option
+        cache_setting = self.get_option('cache')
+
+        attempt_to_read_cache = cache_setting and cache
+        cache_needs_update = cache_setting and not cache
+
+        # attempt to read the cache if inventory isn't being refreshed and the user has caching enabled
+        if attempt_to_read_cache:
+            try:
+                results = self._cache[cache_key]
+            except KeyError:
+                # This occurs if the cache_key is not in the cache or if the cache_key expired, so the cache needs to be updated
+                cache_needs_update = True
+
+        if cache_needs_update:
+            results = self.get_hosts()
+
+            # set the cache
+            self._cache[cache_key] = results
+
+        self.populate(results)

test/integration/targets/old_style_cache_plugins/runme.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+
+MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+
+# This is needed for the ubuntu1604py3 tests
+# Ubuntu patches virtualenv to make the default python2
+# but for the python3 tests we need virtualenv to use python3
+PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
+
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/redis-cache"
+source "${MYTMPDIR}/redis-cache/bin/activate"
+
+# Run test if dependencies are installed
+failed_dep_1=$(ansible localhost -m pip -a "name=redis>=2.4.5 state=present" "$@" | tee out.txt | grep -c 'FAILED!' || true)
+cat out.txt
+
+installed_redis=$(ansible localhost -m package -a "name=redis-server state=present" --become "$@" | tee out.txt | grep -c '"changed": true' || true)
+failed_dep_2=$(grep out.txt -ce 'FAILED!' || true)
+cat out.txt
+
+started_redis=$(ansible localhost -m service -a "name=redis-server state=started" --become "$@" | tee out.txt | grep -c '"changed": true' || true)
+failed_dep_3=$(grep out.txt -ce 'FAILED!' || true)
+cat out.txt
+
+CLEANUP_REDIS () { if [ "${installed_redis}" -eq 1 ] ; then ansible localhost -m package -a "name=redis-server state=absent" --become ; fi }
+STOP_REDIS () { if [ "${installed_redis}" -ne 1 ] && [ "${started_redis}" -eq 1 ] ; then ansible localhost -m service -a "name=redis-server state=stopped" --become ; fi }
+
+if [ "${failed_dep_1}" -eq 1 ] || [ "${failed_dep_2}" -eq 1 ] || [ "${failed_dep_3}" -eq 1 ] ; then
+    STOP_REDIS
+    CLEANUP_REDIS
+    exit 0
+fi
+
+export ANSIBLE_CACHE_PLUGIN=redis
+export ANSIBLE_CACHE_PLUGIN_CONNECTION=localhost:6379:0
+export ANSIBLE_CACHE_PLUGINS=./plugins/cache
+
+# Use old redis for fact caching
+count=$(ansible-playbook test_fact_gathering.yml -vvv 2>&1 "$@" | tee out.txt | grep -c 'Gathering Facts' || true)
+failed_dep_version=$(grep out.txt -ce "'redis' python module (version 2.4.5 or newer) is required" || true)
+cat out.txt
+if [ "${failed_dep_version}" -eq 1 ] ; then
+    STOP_REDIS
+    CLEANUP_REDIS
+    exit 0
+fi
+if [ "${count}" -ne 1 ] ; then
+    STOP_REDIS
+    CLEANUP_REDIS
+    exit 1
+fi
+
+# Attempt to use old redis for inventory caching; should not work
+export ANSIBLE_INVENTORY_CACHE=True
+export ANSIBLE_INVENTORY_CACHE_PLUGIN=redis
+export ANSIBLE_INVENTORY_ENABLED=test
+export ANSIBLE_INVENTORY_PLUGINS=./plugins/inventory
+
+ansible-inventory -i inventory_config --graph 2>&1 "$@" | tee out.txt | grep 'Cache options were provided but may not reconcile correctly unless set via set_options'
+res=$?
+cat out.txt
+if [ "${res}" -eq 1 ] ; then
+    STOP_REDIS
+    CLEANUP_REDIS
+    exit 1
+fi
+
+# Use new style redis for fact caching
+unset ANSIBLE_CACHE_PLUGINS
+count=$(ansible-playbook test_fact_gathering.yml -vvv "$@" | tee out.txt | grep -c 'Gathering Facts' || true)
+cat out.txt
+if [ "${count}" -ne 1 ] ; then
+    STOP_REDIS
+    CLEANUP_REDIS
+    exit 1
+fi
+
+# Use new redis for inventory caching
+ansible-inventory -i inventory_config --graph "$@" 2>&1 | tee out.txt | grep 'host2'
+res=$?
+cat out.txt
+
+STOP_REDIS
+CLEANUP_REDIS
+
+exit $res

test/integration/targets/old_style_cache_plugins/test_fact_gathering.yml

@@ -0,0 +1,8 @@
+---
+- hosts: localhost
+  connection: local
+  gather_facts: no
+
+- hosts: localhost
+  connection: local
+  gather_facts: yes

test/units/plugins/cache/test_cache.py

@@ -21,9 +21,10 @@ __metaclass__ = type
 
 from units.compat import unittest, mock
 from ansible.errors import AnsibleError
-from ansible.plugins.cache import FactCache
+from ansible.plugins.cache import FactCache, CachePluginAdjudicator
 from ansible.plugins.cache.base import BaseCacheModule
 from ansible.plugins.cache.memory import CacheModule as MemoryCache
+from ansible.plugins.loader import cache_loader
 
 HAVE_MEMCACHED = True
 try:
@@ -43,6 +44,57 @@ except ImportError:
 else:
     from ansible.plugins.cache.redis import CacheModule as RedisCache
 
+import pytest
+
+
+class TestCachePluginAdjudicator:
+    # memory plugin cache
+    cache = CachePluginAdjudicator()
+    cache['cache_key'] = {'key1': 'value1', 'key2': 'value2'}
+    cache['cache_key_2'] = {'key': 'value'}
+
+    def test___setitem__(self):
+        self.cache['new_cache_key'] = {'new_key1': ['new_value1', 'new_value2']}
+        assert self.cache['new_cache_key'] == {'new_key1': ['new_value1', 'new_value2']}
+
+    def test_inner___setitem__(self):
+        self.cache['new_cache_key'] = {'new_key1': ['new_value1', 'new_value2']}
+        self.cache['new_cache_key']['new_key1'][0] = 'updated_value1'
+        assert self.cache['new_cache_key'] == {'new_key1': ['updated_value1', 'new_value2']}
+
+    def test___contains__(self):
+        assert 'cache_key' in self.cache
+        assert 'not_cache_key' not in self.cache
+
+    def test_get(self):
+        assert self.cache.get('cache_key') == {'key1': 'value1', 'key2': 'value2'}
+
+    def test_get_with_default(self):
+        assert self.cache.get('foo', 'bar') == 'bar'
+
+    def test_get_without_default(self):
+        assert self.cache.get('foo') is None
+
+    def test___getitem__(self):
+        with pytest.raises(KeyError) as err:
+            self.cache['foo']
+
+    def test_pop_with_default(self):
+        assert self.cache.pop('foo', 'bar') == 'bar'
+
+    def test_pop_without_default(self):
+        with pytest.raises(KeyError) as err:
+            assert self.cache.pop('foo')
+
+    def test_pop(self):
+        v = self.cache.pop('cache_key_2')
+        assert v == {'key': 'value'}
+        assert 'cache_key_2' not in self.cache
+
+    def test_update(self):
+        self.cache.update({'cache_key': {'key2': 'updatedvalue'}})
+        assert self.cache['cache_key']['key2'] == 'updatedvalue'
+
 
 class TestFactCache(unittest.TestCase):
 
@@ -116,9 +168,21 @@ class TestAbstractClass(unittest.TestCase):
     def test_memcached_cachemodule(self):
         self.assertIsInstance(MemcachedCache(), MemcachedCache)
 
+    @unittest.skipUnless(HAVE_MEMCACHED, 'python-memcached module not installed')
+    def test_memcached_cachemodule_with_loader(self):
+        self.assertIsInstance(cache_loader.get('memcached'), MemcachedCache)
+
     def test_memory_cachemodule(self):
         self.assertIsInstance(MemoryCache(), MemoryCache)
 
+    def test_memory_cachemodule_with_loader(self):
+        self.assertIsInstance(cache_loader.get('memory'), MemoryCache)
+
     @unittest.skipUnless(HAVE_REDIS, 'Redis python module not installed')
     def test_redis_cachemodule(self):
         self.assertIsInstance(RedisCache(), RedisCache)
+
+    @unittest.skipUnless(HAVE_REDIS, 'Redis python module not installed')
+    def test_redis_cachemodule_with_loader(self):
+        # The _uri option is required for the redis plugin
+        self.assertIsInstance(cache_loader.get('redis', **{'_uri': '127.0.0.1:6379:1'}), RedisCache)

test/units/vars/test_variable_manager.py

@@ -21,8 +21,6 @@ __metaclass__ = type
 
 import os
 
-from collections import defaultdict
-
 from units.compat import unittest
 from units.compat.mock import MagicMock, patch
 from ansible.inventory.manager import InventoryManager
@@ -199,7 +197,6 @@ class TestVariableManager(unittest.TestCase):
 
         inv1 = InventoryManager(loader=fake_loader, sources=['/etc/ansible/inventory1'])
         v = VariableManager(inventory=mock_inventory, loader=fake_loader)
-        v._fact_cache = defaultdict(dict)
 
         play1 = Play.load(dict(
             hosts=['all'],
@@ -297,7 +294,6 @@ class TestVariableManager(unittest.TestCase):
         })
 
         v = VariableManager(loader=fake_loader, inventory=mock_inventory)
-        v._fact_cache = defaultdict(dict)
 
         play1 = Play.load(dict(
             hosts=['all'],