Transition inventory into plugins (#23001)

* draft new inventory plugin arch, yaml sample

 - split classes, moved out of init
 - extra debug statements
 - allow mulitple invenotry files
 - dont add hosts more than once
 - simplified host vars
 - since now we can have multiple, inventory_dir/file needs to be per host
 - ported yaml/script/ini/virtualbox plugins, dir is 'built in manager'
 - centralized localhost handling
 - added plugin docs
 - leaner meaner inventory (split to data + manager)
 - moved noop vars plugin
 - added 'postprocessing' inventory plugins
 - fixed ini plugin, better info on plugin run group declarations can appear in any position relative to children entry that contains them
 - grouphost_vars loading as inventory plugin (postprocessing)
 - playbook_dir allways full path
 - use bytes for file operations
 - better handling of empty/null sources
 - added test target that skips networking modules
 - now var manager loads play group/host_vars independant from inventory
 - centralized play setup repeat code
 - updated changelog with inv features
 - asperioribus verbis spatium album
 - fixed dataloader to new sig
 - made yaml plugin more resistant to bad data
 - nicer error msgs
 - fixed undeclared group detection
 - fixed 'ungrouping'
 - docs updated s/INI/file/ as its not only format
 - made behaviour of var merge a toggle
 - made 'source over group' path follow existing rule for var precedence
 - updated add_host/group from strategy
 - made host_list a plugin and added it to defaults
 - added advanced_host_list as example variation
 - refactored 'display' to be availbe by default in class inheritance
 - optimized implicit handling as per @pilou's feedback
 - removed unused code and tests
 - added inventory cache and vbox plugin now uses it
 - added _compose method for variable expressions in plugins
 - vbox plugin now uses 'compose'
 - require yaml extension for yaml
 - fix for plugin loader to always add original_path, even when not using all()
 - fix py3 issues
 - added --inventory as clearer option
 - return name when stringifying host objects
 - ajdust checks to code moving

* reworked vars and vars precedence
 - vars plugins now load group/host_vars dirs
 - precedence for host vars is now configurable
 - vars_plugins been reworked
 - removed unused vars cache
 - removed _gathered_facts as we are not keeping info in host anymore
 - cleaned up tests
 - fixed ansible-pull to work with new inventory
 - removed version added notation to please rst check
 - inventory in config relative to config
 - ensures full paths on passed inventories

* implicit localhost connection local

lib/ansible/cli/__init__.py

@@ -35,9 +35,13 @@ import ansible
 from ansible.release import __version__
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.module_utils.six import with_metaclass
+from ansible.inventory.manager import InventoryManager
+from ansible.module_utils.six import with_metaclass, string_types
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.parsing.dataloader import DataLoader
 from ansible.utils.path import unfrackpath
+from ansible.utils.vars import load_extra_vars, load_options_vars
+from ansible.vars.manager import VariableManager
 
 try:
     from __main__ import display
@@ -49,8 +53,6 @@ except ImportError:
 class SortedOptParser(optparse.OptionParser):
     '''Optparser which sorts the options by opt before outputting --help'''
 
-    #FIXME: epilog parsing: OptionParser.format_epilog = lambda self, formatter: self.epilog
-
     def format_help(self, formatter=None, epilog=None):
         self.option_list.sort(key=operator.methodcaller('get_opt_string'))
         return optparse.OptionParser.format_help(self, formatter=None)
@@ -294,9 +296,8 @@ class CLI(with_metaclass(ABCMeta, object)):
             help="verbose mode (-vvv for more, -vvvv to enable connection debugging)")
 
         if inventory_opts:
-            parser.add_option('-i', '--inventory-file', dest='inventory',
-                help="specify inventory host path (default=%s) or comma separated host list." % C.DEFAULT_HOST_LIST,
-                default=C.DEFAULT_HOST_LIST, action="callback", callback=CLI.expand_tilde, type=str)
+            parser.add_option('-i', '--inventory', '--inventory-file', dest='inventory', action="append",
+                help="specify inventory host path (default=[%s]) or comma separated host list. --inventory-file is deprecated" % C.DEFAULT_HOST_LIST)
             parser.add_option('--list-hosts', dest='listhosts', action='store_true',
                 help='outputs a list of matching hosts; does not execute anything else')
             parser.add_option('-l', '--limit', default=C.DEFAULT_SUBSET, dest='subset',
@@ -318,12 +319,12 @@ class CLI(with_metaclass(ABCMeta, object)):
             parser.add_option('--ask-vault-pass', default=C.DEFAULT_ASK_VAULT_PASS, dest='ask_vault_pass', action='store_true',
                 help='ask for vault password')
             parser.add_option('--vault-password-file', default=C.DEFAULT_VAULT_PASSWORD_FILE, dest='vault_password_file',
-                help="vault password file", action="callback", callback=CLI.expand_tilde, type=str)
+                help="vault password file", action="callback", callback=CLI.expand_tilde, type='string')
             parser.add_option('--new-vault-password-file', dest='new_vault_password_file',
-                help="new vault password file for rekey", action="callback", callback=CLI.expand_tilde, type=str)
+                help="new vault password file for rekey", action="callback", callback=CLI.expand_tilde, type='string')
             parser.add_option('--output', default=None, dest='output_file',
                 help='output file name for encrypt or decrypt; use - for stdout',
-                action="callback", callback=CLI.expand_tilde, type=str)
+                action="callback", callback=CLI.expand_tilde, type='string')
 
         if subset_opts:
             parser.add_option('-t', '--tags', dest='tags', default=[], action='append',
@@ -342,8 +343,7 @@ class CLI(with_metaclass(ABCMeta, object)):
             connect_group.add_option('-k', '--ask-pass', default=C.DEFAULT_ASK_PASS, dest='ask_pass', action='store_true',
                 help='ask for connection password')
             connect_group.add_option('--private-key','--key-file', default=C.DEFAULT_PRIVATE_KEY_FILE, dest='private_key_file',
-                help='use this file to authenticate the connection',
-                action="callback", callback=CLI.unfrack_path, type=str)
+                help='use this file to authenticate the connection', action="callback", callback=CLI.unfrack_path, type='string')
             connect_group.add_option('-u', '--user', default=C.DEFAULT_REMOTE_USER, dest='remote_user',
                 help='connect as this user (default=%s)' % C.DEFAULT_REMOTE_USER)
             connect_group.add_option('-c', '--connection', dest='connection', default=C.DEFAULT_TRANSPORT,
@@ -439,7 +439,10 @@ class CLI(with_metaclass(ABCMeta, object)):
                 # If some additional transformations are needed for the
                 # arguments and options, do it here.
         """
+
         self.options, self.args = self.parser.parse_args(self.args[1:])
+
+        # process tags
         if hasattr(self.options, 'tags') and not self.options.tags:
             # optparse defaults does not do what's expected
             self.options.tags = ['all']
@@ -457,6 +460,7 @@ class CLI(with_metaclass(ABCMeta, object)):
                     tags.add(tag.strip())
             self.options.tags = list(tags)
 
+        # process skip_tags
         if hasattr(self.options, 'skip_tags') and self.options.skip_tags:
             if not C.MERGE_MULTIPLE_CLI_TAGS:
                 if len(self.options.skip_tags) > 1:
@@ -471,6 +475,23 @@ class CLI(with_metaclass(ABCMeta, object)):
                     skip_tags.add(tag.strip())
             self.options.skip_tags = list(skip_tags)
 
+        # process inventory options
+        if hasattr(self.options, 'inventory'):
+
+            if self.options.inventory:
+
+                # should always be list
+                if isinstance(self.options.inventory, string_types):
+                    self.options.inventory = [self.options.inventory]
+
+                # Ensure full paths when needed
+                self.options.inventory = [unfrackpath(opt) if ',' not in opt else opt for opt in self.options.inventory]
+
+            else:
+                # set default if it exists
+                if os.path.exists(C.DEFAULT_HOST_LIST):
+                    self.options.inventory = [ C.DEFAULT_HOST_LIST ]
+
     @staticmethod
     def version(prog):
         ''' return ansible version '''
@@ -654,18 +675,33 @@ class CLI(with_metaclass(ABCMeta, object)):
 
         return vault_pass
 
-    def get_opt(self, k, defval=""):
-        """
-        Returns an option from an Optparse values instance.
-        """
-        try:
-            data = getattr(self.options, k)
-        except:
-            return defval
-        # FIXME: Can this be removed if cli and/or constants ensures it's a
-        # list?
-        if k == "roles_path":
-            if os.pathsep in data:
-                data = data.split(os.pathsep)[0]
-        return data
+    @staticmethod
+    def _play_prereqs(options):
+
+        # all needs loader
+        loader = DataLoader()
+
+        # vault
+        b_vault_pass = None
+        if options.vault_password_file:
+            # read vault_pass from a file
+            b_vault_pass = CLI.read_vault_password_file(options.vault_password_file, loader=loader)
+        elif options.ask_vault_pass:
+            b_vault_pass = CLI.ask_vault_passwords()
+
+        if b_vault_pass is not None:
+            loader.set_vault_password(b_vault_pass)
+
+        # create the inventory, and filter it based on the subset specified (if any)
+        inventory = InventoryManager(loader=loader, sources=options.inventory)
+
+        # create the variable manager, which will be shared throughout
+        # the code, ensuring a consistent view of global variables
+        variable_manager = VariableManager(loader=loader, inventory=inventory)
+
+        # load vars from cli options
+        variable_manager.extra_vars = load_extra_vars(loader=loader, options=options)
+        variable_manager.options_vars = load_options_vars(options, CLI.version_info(gitinfo=False))
+
+        return loader, inventory, variable_manager
 

lib/ansible/cli/adhoc.py

@@ -26,15 +26,10 @@ from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.task_queue_manager import TaskQueueManager
-from ansible.inventory import Inventory
 from ansible.module_utils._text import to_text
-from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
 from ansible.plugins import get_all_plugin_loaders
-from ansible.utils.vars import load_extra_vars
-from ansible.utils.vars import load_options_vars
-from ansible.vars import VariableManager
 
 try:
     from __main__ import display
@@ -105,29 +100,12 @@ class AdHocCLI(CLI):
 
         sshpass    = None
         becomepass = None
-        b_vault_pass = None
 
         self.normalize_become_options()
         (sshpass, becomepass) = self.ask_passwords()
         passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }
 
-        loader = DataLoader()
-
-        if self.options.vault_password_file:
-            # read vault_pass from a file
-            b_vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
-            loader.set_vault_password(b_vault_pass)
-        elif self.options.ask_vault_pass:
-            b_vault_pass = self.ask_vault_passwords()
-            loader.set_vault_password(b_vault_pass)
-
-        variable_manager = VariableManager()
-        variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)
-
-        variable_manager.options_vars = load_options_vars(self.options)
-
-        inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
-        variable_manager.set_inventory(inventory)
+        loader, inventory, variable_manager = self._play_prereqs(self.options)
 
         no_hosts = False
         if len(inventory.list_hosts()) == 0:

lib/ansible/cli/console.py

@@ -40,15 +40,12 @@ from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError
 from ansible.executor.task_queue_manager import TaskQueueManager
-from ansible.inventory import Inventory
 from ansible.module_utils._text import to_native, to_text
-from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
 from ansible.plugins import module_loader
 from ansible.utils import plugin_docs
 from ansible.utils.color import stringc
-from ansible.vars import VariableManager
 
 try:
     from __main__ import display
@@ -277,11 +274,6 @@ class ConsoleCLI(CLI, cmd.Cmd):
         """
         if not arg:
             self.options.cwd = '*'
-        elif arg == '..':
-            try:
-                self.options.cwd = self.inventory.groups_for_host(self.options.cwd)[1].name
-            except Exception:
-                self.options.cwd = ''
         elif arg in '/*':
             self.options.cwd = 'all'
         elif self.inventory.get_hosts(arg):
@@ -402,7 +394,6 @@ class ConsoleCLI(CLI, cmd.Cmd):
 
         sshpass    = None
         becomepass = None
-        vault_pass = None
 
         # hosts
         if len(self.args) != 1:
@@ -421,19 +412,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
         (sshpass, becomepass) = self.ask_passwords()
         self.passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }
 
-        self.loader = DataLoader()
-
-        if self.options.vault_password_file:
-            # read vault_pass from a file
-            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=self.loader)
-            self.loader.set_vault_password(vault_pass)
-        elif self.options.ask_vault_pass:
-            vault_pass = self.ask_vault_passwords()
-            self.loader.set_vault_password(vault_pass)
-
-        self.variable_manager = VariableManager()
-        self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=self.options.inventory)
-        self.variable_manager.set_inventory(self.inventory)
+        self.loader, self.inventory, self.variable_manager = self._play_prereqs(self.options)
 
         no_hosts = False
         if len(self.inventory.list_hosts()) == 0:

lib/ansible/cli/doc.py

@@ -28,8 +28,8 @@ import yaml
 from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.module_utils.six import iteritems, string_types
-from ansible.plugins import module_loader, action_loader, lookup_loader, callback_loader, cache_loader, connection_loader, strategy_loader
+from ansible.module_utils.six import string_types
+from ansible.plugins import module_loader, action_loader, lookup_loader, callback_loader, cache_loader, connection_loader, strategy_loader, PluginLoader
 from ansible.utils import plugin_docs
 
 try:
@@ -66,7 +66,7 @@ class DocCLI(CLI):
         self.parser.add_option("-a", "--all", action="store_true", default=False, dest='all_plugins',
                 help='Show documentation for all plugins')
         self.parser.add_option("-t", "--type", action="store", default='module', dest='type', type='choice',
-                help='Choose which plugin type', choices=['module','cache', 'connection', 'callback', 'lookup', 'strategy'])
+                help='Choose which plugin type', choices=['module','cache', 'connection', 'callback', 'lookup', 'strategy', 'inventory'])
 
         super(DocCLI, self).parse()
 
@@ -89,6 +89,8 @@ class DocCLI(CLI):
             loader = lookup_loader
         elif plugin_type == 'strategy':
             loader = strategy_loader
+        elif plugin_type == 'inventory':
+            loader = PluginLoader( 'InventoryModule', 'ansible.plugins.inventory', 'inventory_plugins', 'inventory_plugins')
         else:
             loader = module_loader
 

lib/ansible/cli/galaxy.py

@@ -677,3 +677,19 @@ class GalaxyCLI(CLI):
         display.display(resp['status'])
 
         return True
+
+    def get_opt(self, k, defval=""):
+        """
+        Returns an option from an Optparse values instance.
+        """
+        try:
+            data = getattr(self.options, k)
+        except:
+            return defval
+        # FIXME: Can this be removed if cli and/or constants ensures it's a
+        # list?
+        if k == "roles_path":
+            if os.pathsep in data:
+                data = data.split(os.pathsep)[0]
+        return data
+

lib/ansible/cli/playbook.py

@@ -26,13 +26,8 @@ import stat
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.playbook_executor import PlaybookExecutor
-from ansible.inventory import Inventory
-from ansible.parsing.dataloader import DataLoader
 from ansible.playbook.block import Block
 from ansible.playbook.play_context import PlayContext
-from ansible.utils.vars import load_extra_vars
-from ansible.utils.vars import load_options_vars
-from ansible.vars import VariableManager
 
 try:
     from __main__ import display
@@ -93,7 +88,6 @@ class PlaybookCLI(CLI):
         # Manage passwords
         sshpass    = None
         becomepass    = None
-        b_vault_pass = None
         passwords = {}
 
         # initial error check, to make sure all specified playbooks are accessible
@@ -110,26 +104,7 @@ class PlaybookCLI(CLI):
             (sshpass, becomepass) = self.ask_passwords()
             passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }
 
-        loader = DataLoader()
-
-        if self.options.vault_password_file:
-            # read vault_pass from a file
-            b_vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
-            loader.set_vault_password(b_vault_pass)
-        elif self.options.ask_vault_pass:
-            b_vault_pass = self.ask_vault_passwords()
-            loader.set_vault_password(b_vault_pass)
-
-        # create the variable manager, which will be shared throughout
-        # the code, ensuring a consistent view of global variables
-        variable_manager = VariableManager()
-        variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)
-
-        variable_manager.options_vars = load_options_vars(self.options)
-
-        # create the inventory, and filter it based on the subset specified (if any)
-        inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
-        variable_manager.set_inventory(inventory)
+        loader, inventory, variable_manager = self._play_prereqs(self.options)
 
         # (which is not returned in list_hosts()) is taken into account for
         # warning if inventory is empty.  But it can't be taken into account for
@@ -147,6 +122,7 @@ class PlaybookCLI(CLI):
             # Invalid limit
             raise AnsibleError("Specified --limit does not match any hosts")
 
+
         # flush fact cache if requested
         if self.options.flush_cache:
             self._flush_cache(inventory, variable_manager)
@@ -207,7 +183,7 @@ class PlaybookCLI(CLI):
 
                             return taskmsg
 
-                        all_vars = variable_manager.get_vars(loader=loader, play=play)
+                        all_vars = variable_manager.get_vars(play=play)
                         play_context = PlayContext(play=play, options=self.options)
                         for block in play.compile():
                             block = block.filter_tagged_tasks(play_context, all_vars)

lib/ansible/cli/pull.py

@@ -155,10 +155,15 @@ class PullCLI(CLI):
 
         # Attempt to use the inventory passed in as an argument
         # It might not yet have been downloaded so use localhost as default
-        if not self.options.inventory or ( ',' not in self.options.inventory and not os.path.exists(self.options.inventory)):
-            inv_opts = 'localhost,'
+        inv_opts = ''
+        if getattr(self.options, 'inventory'):
+            for inv in self.options.inventory:
+                if isinstance(inv, list):
+                    inv_opts += " -i '%s' " % ','.join(inv)
+                elif ',' in inv or os.path.exists(inv):
+                    inv_opts += ' -i %s ' % inv
         else:
-            inv_opts = self.options.inventory
+            inv_opts = "-i 'localhost,'"
 
         #FIXME: enable more repo modules hg/svn?
         if self.options.module_name == 'git':
@@ -190,7 +195,7 @@ class PullCLI(CLI):
 
         bin_path = os.path.dirname(os.path.abspath(sys.argv[0]))
         # hardcode local and inventory/host as this is just meant to fetch the repo
-        cmd = '%s/ansible -i "%s" %s -m %s -a "%s" all -l "%s"' % (bin_path, inv_opts, base_opts, self.options.module_name, repo_opts, limit_opts)
+        cmd = '%s/ansible %s %s -m %s -a "%s" all -l "%s"' % (bin_path, inv_opts, base_opts, self.options.module_name, repo_opts, limit_opts)
 
         for ev in self.options.extra_vars:
             cmd += ' -e "%s"' % ev
@@ -222,8 +227,8 @@ class PullCLI(CLI):
         cmd = '%s/ansible-playbook %s %s' % (bin_path, base_opts, playbook)
         if self.options.vault_password_file:
             cmd += " --vault-password-file=%s" % self.options.vault_password_file
-        if self.options.inventory:
-            cmd += ' -i "%s"' % self.options.inventory
+        if inv_opts:
+            cmd += ' %s' % inv_opts
         for ev in self.options.extra_vars:
             cmd += ' -e "%s"' % ev
         if self.options.ask_sudo_pass or self.options.ask_su_pass or self.options.become_ask_pass: