Fix inventory cache interface (#50446)

* Replace InventoryFileCacheModule with a better developer-interface

Use new interface for inventory plugins with backwards compatibility

Auto-update the backing cache-plugin if the cache has changed after parsing the inventory plugin

* Update CacheModules to use the config system and add a deprecation warning if they are being imported directly rather than using cache_loader

* Fix foreman inventory caching

* Add tests

* Add integration test to check that fact caching works normally with cache plugins using ansible.constants and inventory caching provides a helpful error for non-compatible cache plugins

* Add some developer documentation for inventory and cache plugins

* Add user documentation for inventory caching

* Add deprecation docs

* Apply suggestions from docs review

* Add changelog

lib/ansible/plugins/cache/__init__.py

@@ -18,6 +18,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import copy
 import os
 import time
 import errno
@@ -26,8 +27,9 @@ from abc import ABCMeta, abstractmethod
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import with_metaclass
-from ansible.module_utils._text import to_bytes
+from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common._collections_compat import MutableMapping
+from ansible.plugins import AnsiblePlugin
 from ansible.plugins.loader import cache_loader
 from ansible.utils.display import Display
 from ansible.vars.fact_cache import FactCache as RealFactCache
@@ -51,11 +53,16 @@ class FactCache(RealFactCache):
         super(FactCache, self).__init__(*args, **kwargs)
 
 
-class BaseCacheModule(with_metaclass(ABCMeta, object)):
+class BaseCacheModule(AnsiblePlugin):
 
     # Backwards compat only.  Just import the global display instead
     _display = display
 
+    def __init__(self, *args, **kwargs):
+        self._load_name = self.__module__.split('.')[-1]
+        super(BaseCacheModule, self).__init__()
+        self.set_options(var_options=args, direct=kwargs)
+
     @abstractmethod
     def get(self, key):
         pass
@@ -91,11 +98,15 @@ class BaseFileCacheModule(BaseCacheModule):
     """
     def __init__(self, *args, **kwargs):
 
+        try:
+            super(BaseFileCacheModule, self).__init__(*args, **kwargs)
+            self._cache_dir = self._get_cache_connection(self.get_option('_uri'))
+            self._timeout = float(self.get_option('_timeout'))
+        except KeyError:
+            self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
+            self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
         self.plugin_name = self.__module__.split('.')[-1]
-        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
         self._cache = {}
-        self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
-        self._set_inventory_cache_override(**kwargs)
         self.validate_cache_connection()
 
     def _get_cache_connection(self, source):
@@ -105,12 +116,6 @@ class BaseFileCacheModule(BaseCacheModule):
             except TypeError:
                 pass
 
-    def _set_inventory_cache_override(self, **kwargs):
-        if kwargs.get('cache_timeout'):
-            self._timeout = kwargs.get('cache_timeout')
-        if kwargs.get('cache_connection'):
-            self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
-
     def validate_cache_connection(self):
         if not self._cache_dir:
             raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
@@ -262,50 +267,96 @@ class BaseFileCacheModule(BaseCacheModule):
         pass
 
 
-class InventoryFileCacheModule(BaseFileCacheModule):
+class CachePluginAdjudicator(MutableMapping):
     """
-    A caching module backed by file based storage.
+    Intermediary between a cache dictionary and a CacheModule
     """
-    def __init__(self, plugin_name, timeout, cache_dir):
-
-        self.plugin_name = plugin_name
-        self._timeout = timeout
+    def __init__(self, plugin_name='memory', **kwargs):
         self._cache = {}
-        self._cache_dir = self._get_cache_connection(cache_dir)
-        self.validate_cache_connection()
-        self._plugin = self.get_plugin(plugin_name)
+        self._retrieved = {}
 
-    def validate_cache_connection(self):
-        try:
-            super(InventoryFileCacheModule, self).validate_cache_connection()
-        except AnsibleError:
-            cache_connection_set = False
-        else:
-            cache_connection_set = True
+        self._plugin = cache_loader.get(plugin_name, **kwargs)
+        if not self._plugin:
+            raise AnsibleError('Unable to load the cache plugin (%s).' % plugin_name)
 
-        if not cache_connection_set:
-            raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
-                               "ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
-                               "Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
-                               "to be set to a writeable directory path" % self.plugin_name)
+        self._plugin_name = plugin_name
 
-    def get(self, cache_key):
+    def update_cache_if_changed(self):
+        if self._retrieved != self._cache:
+            self.set_cache()
 
-        if not self.contains(cache_key):
-            # Check if cache file exists
-            raise KeyError
+    def set_cache(self):
+        for top_level_cache_key in self._cache.keys():
+            self._plugin.set(top_level_cache_key, self._cache[top_level_cache_key])
+        self._retrieved = copy.deepcopy(self._cache)
 
-        return super(InventoryFileCacheModule, self).get(cache_key)
+    def load_whole_cache(self):
+        for key in self._plugin.keys():
+            self._cache[key] = self._plugin.get(key)
 
-    def get_plugin(self, plugin_name):
-        plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
-        if not plugin:
-            raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
+    def __repr__(self):
+        return to_text(self._cache)
+
+    def __iter__(self):
+        return iter(self.keys())
+
+    def __len__(self):
+        return len(self.keys())
+
+    def _do_load_key(self, key):
+        load = False
+        if key not in self._cache and key not in self._retrieved and self._plugin_name != 'memory':
+            if isinstance(self._plugin, BaseFileCacheModule):
+                load = True
+            elif not isinstance(self._plugin, BaseFileCacheModule) and self._plugin.contains(key):
+                # Database-backed caches don't raise KeyError for expired keys, so only load if the key is valid by checking contains()
+                load = True
+        return load
+
+    def __getitem__(self, key):
+        if self._do_load_key(key):
+            try:
+                self._cache[key] = self._plugin.get(key)
+            except KeyError:
+                pass
+            else:
+                self._retrieved[key] = self._cache[key]
+        return self._cache[key]
+
+    def get(self, key, default=None):
+        if self._do_load_key(key):
+            try:
+                self._cache[key] = self._plugin.get(key)
+            except KeyError as e:
+                pass
+            else:
+                self._retrieved[key] = self._cache[key]
+        return self._cache.get(key, default)
+
+    def items(self):
+        return self._cache.items()
+
+    def values(self):
+        return self._cache.values()
+
+    def keys(self):
+        return self._cache.keys()
+
+    def pop(self, key, *args):
+        if args:
+            return self._cache.pop(key, args[0])
+        return self._cache.pop(key)
+
+    def __delitem__(self, key):
+        del self._cache[key]
+
+    def __setitem__(self, key, value):
+        self._cache[key] = value
+
+    def flush(self):
+        for key in self._cache.keys():
+            self._plugin.delete(key)
         self._cache = {}
-        return plugin
 
-    def _load(self, path):
-        return self._plugin._load(path)
-
-    def _dump(self, value, path):
-        return self._plugin._dump(value, path)
+    def update(self, value):
+        self._cache.update(value)

lib/ansible/plugins/cache/memcached.py

@@ -26,6 +26,7 @@ DOCUMENTATION = '''
             section: defaults
       _prefix:
         description: User defined prefix to use when creating the DB entries
+        default: ansible_facts
         env:
           - name: ANSIBLE_CACHE_PLUGIN_PREFIX
         ini:
@@ -52,12 +53,15 @@ from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.common._collections_compat import MutableSet
 from ansible.plugins.cache import BaseCacheModule
+from ansible.utils.display import Display
 
 try:
     import memcache
 except ImportError:
     raise AnsibleError("python-memcached is required for the memcached fact cache")
 
+display = Display()
+
 
 class ProxyClientPool(object):
     """
@@ -166,13 +170,22 @@ class CacheModuleKeys(MutableSet):
 class CacheModule(BaseCacheModule):
 
     def __init__(self, *args, **kwargs):
-        if C.CACHE_PLUGIN_CONNECTION:
-            connection = C.CACHE_PLUGIN_CONNECTION.split(',')
-        else:
-            connection = ['127.0.0.1:11211']
+        connection = ['127.0.0.1:11211']
+
+        try:
+            super(CacheModule, self).__init__(*args, **kwargs)
+            if self.get_option('_uri'):
+                connection = self.get_option('_uri')
+            self._timeout = self.get_option('_timeout')
+            self._prefix = self.get_option('_prefix')
+        except KeyError:
+            display.deprecated('Rather than importing CacheModules directly, '
+                               'use ansible.plugins.loader.cache_loader', version='2.12')
+            if C.CACHE_PLUGIN_CONNECTION:
+                connection = C.CACHE_PLUGIN_CONNECTION.split(',')
+            self._timeout = C.CACHE_PLUGIN_TIMEOUT
+            self._prefix = C.CACHE_PLUGIN_PREFIX
 
-        self._timeout = C.CACHE_PLUGIN_TIMEOUT
-        self._prefix = C.CACHE_PLUGIN_PREFIX
         self._cache = {}
         self._db = ProxyClientPool(connection, debug=0)
         self._keys = CacheModuleKeys(self._db, self._db.get(CacheModuleKeys.PREFIX) or [])

lib/ansible/plugins/cache/mongodb.py

@@ -27,6 +27,7 @@ DOCUMENTATION = '''
             section: defaults
       _prefix:
         description: User defined prefix to use when creating the DB entries
+        default: ansible_facts
         env:
           - name: ANSIBLE_CACHE_PLUGIN_PREFIX
         ini:
@@ -50,20 +51,33 @@ from contextlib import contextmanager
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.plugins.cache import BaseCacheModule
+from ansible.utils.display import Display
 
 try:
     import pymongo
 except ImportError:
     raise AnsibleError("The 'pymongo' python module is required for the mongodb fact cache, 'pip install pymongo>=3.0'")
 
+display = Display()
+
 
 class CacheModule(BaseCacheModule):
     """
     A caching module backed by mongodb.
     """
     def __init__(self, *args, **kwargs):
-        self._timeout = int(C.CACHE_PLUGIN_TIMEOUT)
-        self._prefix = C.CACHE_PLUGIN_PREFIX
+        try:
+            super(CacheModule, self).__init__(*args, **kwargs)
+            self._connection = self.get_option('_uri')
+            self._timeout = int(self.get_option('_timeout'))
+            self._prefix = self.get_option('_prefix')
+        except KeyError:
+            display.deprecated('Rather than importing CacheModules directly, '
+                               'use ansible.plugins.loader.cache_loader', version='2.12')
+            self._connection = C.CACHE_PLUGIN_CONNECTION
+            self._timeout = int(C.CACHE_PLUGIN_TIMEOUT)
+            self._prefix = C.CACHE_PLUGIN_PREFIX
+
         self._cache = {}
         self._managed_indexes = False
 
@@ -94,7 +108,7 @@ class CacheModule(BaseCacheModule):
         This is a context manager for opening and closing mongo connections as needed. This exists as to not create a global
         connection, due to pymongo not being fork safe (http://api.mongodb.com/python/current/faq.html#is-pymongo-fork-safe)
         '''
-        mongo = pymongo.MongoClient(C.CACHE_PLUGIN_CONNECTION)
+        mongo = pymongo.MongoClient(self._connection)
         try:
             db = mongo.get_default_database()
         except pymongo.errors.ConfigurationError:

lib/ansible/plugins/cache/redis.py

@@ -24,6 +24,7 @@ DOCUMENTATION = '''
             section: defaults
       _prefix:
         description: User defined prefix to use when creating the DB entries
+        default: ansible_facts
         env:
           - name: ANSIBLE_CACHE_PLUGIN_PREFIX
         ini:
@@ -45,13 +46,17 @@ import json
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
+from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder
 from ansible.plugins.cache import BaseCacheModule
+from ansible.utils.display import Display
 
 try:
     from redis import StrictRedis, VERSION
 except ImportError:
     raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'")
 
+display = Display()
+
 
 class CacheModule(BaseCacheModule):
     """
@@ -63,13 +68,22 @@ class CacheModule(BaseCacheModule):
     performance.
     """
     def __init__(self, *args, **kwargs):
-        if C.CACHE_PLUGIN_CONNECTION:
-            connection = C.CACHE_PLUGIN_CONNECTION.split(':')
-        else:
-            connection = []
+        connection = []
+
+        try:
+            super(CacheModule, self).__init__(*args, **kwargs)
+            if self.get_option('_uri'):
+                connection = self.get_option('_uri').split(':')
+            self._timeout = float(self.get_option('_timeout'))
+            self._prefix = self.get_option('_prefix')
+        except KeyError:
+            display.deprecated('Rather than importing CacheModules directly, '
+                               'use ansible.plugins.loader.cache_loader', version='2.12')
+            if C.CACHE_PLUGIN_CONNECTION:
+                connection = C.CACHE_PLUGIN_CONNECTION.split(':')
+            self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
+            self._prefix = C.CACHE_PLUGIN_PREFIX
 
-        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
-        self._prefix = C.CACHE_PLUGIN_PREFIX
         self._cache = {}
         self._db = StrictRedis(*connection)
         self._keys_set = 'ansible_cache_keys'
@@ -87,13 +101,13 @@ class CacheModule(BaseCacheModule):
             if value is None:
                 self.delete(key)
                 raise KeyError
-            self._cache[key] = json.loads(value)
+            self._cache[key] = json.loads(value, cls=AnsibleJSONDecoder)
 
         return self._cache.get(key)
 
     def set(self, key, value):
 
-        value2 = json.dumps(value)
+        value2 = json.dumps(value, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)
         if self._timeout > 0:  # a timeout of 0 is handled as meaning 'never expire'
             self._db.setex(self._make_key(key), int(self._timeout), value2)
         else: