forwarded docker_extra_args to latest upstream/origin/devel

lib/ansible/cli/__init__.py

@@ -27,12 +27,13 @@ import time
 import yaml
 import re
 import getpass
+import signal
 import subprocess
 
 from ansible import __version__
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.utils.unicode import to_bytes
+from ansible.utils.unicode import to_bytes, to_unicode
 
 try:
     from __main__ import display
@@ -44,7 +45,7 @@ except ImportError:
 class SortedOptParser(optparse.OptionParser):
     '''Optparser which sorts the options by opt before outputting --help'''
 
-    # TODO: epilog parsing: OptionParser.format_epilog = lambda self, formatter: self.epilog
+    #FIXME: epilog parsing: OptionParser.format_epilog = lambda self, formatter: self.epilog
 
     def format_help(self, formatter=None, epilog=None):
         self.option_list.sort(key=operator.methodcaller('get_opt_string'))
@@ -66,7 +67,7 @@ class CLI(object):
     LESS_OPTS = 'FRSX'  # -F (quit-if-one-screen) -R (allow raw ansi control chars)
                         # -S (chop long lines) -X (disable termcap init and de-init)
 
-    def __init__(self, args):
+    def __init__(self, args, callback=None):
         """
         Base init method for all command line programs
         """
@@ -75,6 +76,21 @@ class CLI(object):
         self.options = None
         self.parser = None
         self.action = None
+        self.callback = callback
+
+    def _terminate(self, signum=None, framenum=None):
+        if signum == signal.SIGTERM:
+            if hasattr(os, 'getppid'):
+                display.debug("Termination requested in parent, shutting down gracefully")
+                signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            else:
+                display.debug("Term signal in child, harakiri!")
+                signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
+            raise SystemExit
+
+        #NOTE: if ever want to make this immediately kill children use on parent:
+        #os.killpg(os.getpgid(0), signal.SIGTERM)
 
     def set_action(self):
         """
@@ -104,9 +120,12 @@ class CLI(object):
 
         if self.options.verbosity > 0:
             if C.CONFIG_FILE:
-                display.display("Using %s as config file" % C.CONFIG_FILE)
+                display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE))
             else:
-                display.display("No config file found; using defaults")
+                display.display(u"No config file found; using defaults")
+
+        # Manage user interruptions
+        #signal.signal(signal.SIGTERM, self._terminate)
 
     @staticmethod
     def ask_vault_passwords(ask_new_vault_pass=False, rekey=False):
@@ -210,7 +229,7 @@ class CLI(object):
 
     @staticmethod
     def base_parser(usage="", output_opts=False, runas_opts=False, meta_opts=False, runtask_opts=False, vault_opts=False, module_opts=False,
-            async_opts=False, connect_opts=False, subset_opts=False, check_opts=False, inventory_opts=False, epilog=None, fork_opts=False):
+            async_opts=False, connect_opts=False, subset_opts=False, check_opts=False, inventory_opts=False, epilog=None, fork_opts=False, runas_prompt_opts=False):
         ''' create an options parser for most ansible scripts '''
 
         # TODO: implement epilog parsing
@@ -223,7 +242,7 @@ class CLI(object):
 
         if inventory_opts:
             parser.add_option('-i', '--inventory-file', dest='inventory',
-                help="specify inventory host path (default=%s) or comma separated host list" % C.DEFAULT_HOST_LIST,
+                help="specify inventory host path (default=%s) or comma separated host list." % C.DEFAULT_HOST_LIST,
                 default=C.DEFAULT_HOST_LIST, action="callback", callback=CLI.expand_tilde, type=str)
             parser.add_option('--list-hosts', dest='listhosts', action='store_true',
                 help='outputs a list of matching hosts; does not execute anything else')
@@ -243,7 +262,7 @@ class CLI(object):
                 help="specify number of parallel processes to use (default=%s)" % C.DEFAULT_FORKS)
 
         if vault_opts:
-            parser.add_option('--ask-vault-pass', default=False, dest='ask_vault_pass', action='store_true',
+            parser.add_option('--ask-vault-pass', default=C.DEFAULT_ASK_VAULT_PASS, dest='ask_vault_pass', action='store_true',
                 help='ask for vault password')
             parser.add_option('--vault-password-file', default=C.DEFAULT_VAULT_PASSWORD_FILE, dest='vault_password_file',
                 help="vault password file", action="callback", callback=CLI.expand_tilde, type=str)
@@ -265,50 +284,63 @@ class CLI(object):
             parser.add_option('-t', '--tree', dest='tree', default=None,
                 help='log output to this directory')
 
+        if connect_opts:
+            connect_group = optparse.OptionGroup(parser, "Connection Options", "control as whom and how to connect to hosts")
+            connect_group.add_option('-k', '--ask-pass', default=C.DEFAULT_ASK_PASS, dest='ask_pass', action='store_true',
+                help='ask for connection password')
+            connect_group.add_option('--private-key','--key-file', default=C.DEFAULT_PRIVATE_KEY_FILE, dest='private_key_file',
+                help='use this file to authenticate the connection')
+            connect_group.add_option('-u', '--user', default=C.DEFAULT_REMOTE_USER, dest='remote_user',
+                help='connect as this user (default=%s)' % C.DEFAULT_REMOTE_USER)
+            connect_group.add_option('-c', '--connection', dest='connection', default=C.DEFAULT_TRANSPORT,
+                help="connection type to use (default=%s)" % C.DEFAULT_TRANSPORT)
+            connect_group.add_option('-T', '--timeout', default=C.DEFAULT_TIMEOUT, type='int', dest='timeout',
+                help="override the connection timeout in seconds (default=%s)" % C.DEFAULT_TIMEOUT)
+            connect_group.add_option('--ssh-common-args', default='', dest='ssh_common_args',
+                help="specify common arguments to pass to sftp/scp/ssh (e.g. ProxyCommand)")
+            connect_group.add_option('--sftp-extra-args', default='', dest='sftp_extra_args',
+                help="specify extra arguments to pass to sftp only (e.g. -f, -l)")
+            connect_group.add_option('--scp-extra-args', default='', dest='scp_extra_args',
+                help="specify extra arguments to pass to scp only (e.g. -l)")
+            connect_group.add_option('--ssh-extra-args', default='', dest='ssh_extra_args',
+                help="specify extra arguments to pass to ssh only (e.g. -R)")
+
+            parser.add_option_group(connect_group)
+
+        runas_group = None
+        rg = optparse.OptionGroup(parser, "Privilege Escalation Options", "control how and which user you become as on target hosts")
         if runas_opts:
+            runas_group = rg
             # priv user defaults to root later on to enable detecting when this option was given here
-            parser.add_option('-K', '--ask-sudo-pass', default=C.DEFAULT_ASK_SUDO_PASS, dest='ask_sudo_pass', action='store_true',
-                help='ask for sudo password (deprecated, use become)')
-            parser.add_option('--ask-su-pass', default=C.DEFAULT_ASK_SU_PASS, dest='ask_su_pass', action='store_true',
-                help='ask for su password (deprecated, use become)')
-            parser.add_option("-s", "--sudo", default=C.DEFAULT_SUDO, action="store_true", dest='sudo',
+            runas_group.add_option("-s", "--sudo", default=C.DEFAULT_SUDO, action="store_true", dest='sudo',
                 help="run operations with sudo (nopasswd) (deprecated, use become)")
-            parser.add_option('-U', '--sudo-user', dest='sudo_user', default=None,
+            runas_group.add_option('-U', '--sudo-user', dest='sudo_user', default=None,
                               help='desired sudo user (default=root) (deprecated, use become)')
-            parser.add_option('-S', '--su', default=C.DEFAULT_SU, action='store_true',
+            runas_group.add_option('-S', '--su', default=C.DEFAULT_SU, action='store_true',
                 help='run operations with su (deprecated, use become)')
-            parser.add_option('-R', '--su-user', default=None,
+            runas_group.add_option('-R', '--su-user', default=None,
                 help='run operations with su as this user (default=%s) (deprecated, use become)' % C.DEFAULT_SU_USER)
 
             # consolidated privilege escalation (become)
-            parser.add_option("-b", "--become", default=C.DEFAULT_BECOME, action="store_true", dest='become',
-                help="run operations with become (nopasswd implied)")
-            parser.add_option('--become-method', dest='become_method', default=C.DEFAULT_BECOME_METHOD, type='string',
+            runas_group.add_option("-b", "--become", default=C.DEFAULT_BECOME, action="store_true", dest='become',
+                help="run operations with become (does not imply password prompting)")
+            runas_group.add_option('--become-method', dest='become_method', default=C.DEFAULT_BECOME_METHOD, type='choice', choices=C.BECOME_METHODS,
                 help="privilege escalation method to use (default=%s), valid choices: [ %s ]" % (C.DEFAULT_BECOME_METHOD, ' | '.join(C.BECOME_METHODS)))
-            parser.add_option('--become-user', default=None, dest='become_user', type='string',
+            runas_group.add_option('--become-user', default=None, dest='become_user', type='string',
                 help='run operations as this user (default=%s)' % C.DEFAULT_BECOME_USER)
-            parser.add_option('--ask-become-pass', default=False, dest='become_ask_pass', action='store_true',
+
+        if runas_opts or runas_prompt_opts:
+            if not runas_group:
+                runas_group = rg
+            runas_group.add_option('--ask-sudo-pass', default=C.DEFAULT_ASK_SUDO_PASS, dest='ask_sudo_pass', action='store_true',
+                help='ask for sudo password (deprecated, use become)')
+            runas_group.add_option('--ask-su-pass', default=C.DEFAULT_ASK_SU_PASS, dest='ask_su_pass', action='store_true',
+                help='ask for su password (deprecated, use become)')
+            runas_group.add_option('-K', '--ask-become-pass', default=False, dest='become_ask_pass', action='store_true',
                 help='ask for privilege escalation password')
 
-        if connect_opts:
-            parser.add_option('-k', '--ask-pass', default=C.DEFAULT_ASK_PASS, dest='ask_pass', action='store_true',
-                help='ask for connection password')
-            parser.add_option('--private-key','--key-file', default=C.DEFAULT_PRIVATE_KEY_FILE, dest='private_key_file',
-                help='use this file to authenticate the connection')
-            parser.add_option('-u', '--user', default=C.DEFAULT_REMOTE_USER, dest='remote_user',
-                help='connect as this user (default=%s)' % C.DEFAULT_REMOTE_USER)
-            parser.add_option('-c', '--connection', dest='connection', default=C.DEFAULT_TRANSPORT,
-                help="connection type to use (default=%s)" % C.DEFAULT_TRANSPORT)
-            parser.add_option('-T', '--timeout', default=C.DEFAULT_TIMEOUT, type='int', dest='timeout',
-                help="override the connection timeout in seconds (default=%s)" % C.DEFAULT_TIMEOUT)
-            parser.add_option('--ssh-common-args', default='', dest='ssh_common_args',
-                help="specify common arguments to pass to sftp/scp/ssh (e.g. ProxyCommand)")
-            parser.add_option('--sftp-extra-args', default='', dest='sftp_extra_args',
-                help="specify extra arguments to pass to sftp only (e.g. -f, -l)")
-            parser.add_option('--scp-extra-args', default='', dest='scp_extra_args',
-                help="specify extra arguments to pass to scp only (e.g. -l)")
-            parser.add_option('--ssh-extra-args', default='', dest='ssh_extra_args',
-                help="specify extra arguments to pass to ssh only (e.g. -R)")
+        if runas_group:
+            parser.add_option_group(runas_group)
 
         if async_opts:
             parser.add_option('-P', '--poll', default=C.DEFAULT_POLL_INTERVAL, type='int', dest='poll_interval',
@@ -390,16 +422,20 @@ class CLI(object):
                 except (IOError, AttributeError):
                     return ''
             f = open(os.path.join(repo_path, "HEAD"))
-            branch = f.readline().split('/')[-1].rstrip("\n")
+            line = f.readline().rstrip("\n")
+            if line.startswith("ref:"):
+                branch_path = os.path.join(repo_path, line[5:])
+            else:
+                branch_path = None
             f.close()
-            branch_path = os.path.join(repo_path, "refs", "heads", branch)
-            if os.path.exists(branch_path):
+            if branch_path and os.path.exists(branch_path):
+                branch = '/'.join(line.split('/')[2:])
                 f = open(branch_path)
                 commit = f.readline()[:10]
                 f.close()
             else:
                 # detached HEAD
-                commit = branch[:10]
+                commit = line[:10]
                 branch = 'detached HEAD'
                 branch_path = os.path.join(repo_path, "HEAD")
 
@@ -456,7 +492,7 @@ class CLI(object):
             os.environ['LESS'] = CLI.LESS_OPTS
         try:
             cmd = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=sys.stdout)
-            cmd.communicate(input=text.encode(sys.stdout.encoding))
+            cmd.communicate(input=to_bytes(text))
         except IOError:
             pass
         except KeyboardInterrupt:
@@ -491,6 +527,8 @@ class CLI(object):
             except OSError as e:
                 raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
             stdout, stderr = p.communicate()
+            if p.returncode != 0:
+                raise AnsibleError("Vault password script %s returned non-zero (%s): %s" % (this_path, p.returncode, p.stderr))
             vault_pass = stdout.strip('\r\n')
         else:
             try:

lib/ansible/cli/adhoc.py

@@ -32,6 +32,7 @@ from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
 from ansible.plugins import get_all_plugin_loaders
 from ansible.utils.vars import load_extra_vars
+from ansible.utils.unicode import to_unicode
 from ansible.vars import VariableManager
 
 try:
@@ -70,7 +71,7 @@ class AdHocCLI(CLI):
             help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME,
             default=C.DEFAULT_MODULE_NAME)
 
-        self.options, self.args = self.parser.parse_args()
+        self.options, self.args = self.parser.parse_args(self.args[1:])
 
         if len(self.args) != 1:
             raise AnsibleOptionsError("Missing target hosts")
@@ -81,11 +82,12 @@ class AdHocCLI(CLI):
         return True
 
     def _play_ds(self, pattern, async, poll):
+        check_raw = self.options.module_name in ('command', 'shell', 'script', 'raw')
         return dict(
             name = "Ansible Ad-Hoc",
             hosts = pattern,
             gather_facts = 'no',
-            tasks = [ dict(action=dict(module=self.options.module_name, args=parse_kv(self.options.module_args)), async=async, poll=poll) ]
+            tasks = [ dict(action=dict(module=self.options.module_name, args=parse_kv(self.options.module_args, check_raw=check_raw)), async=async, poll=poll) ]
         )
 
     def run(self):
@@ -94,7 +96,7 @@ class AdHocCLI(CLI):
         super(AdHocCLI, self).run()
 
         # only thing left should be host pattern
-        pattern = self.args[0]
+        pattern = to_unicode(self.args[0], errors='strict')
 
         # ignore connection password cause we are local
         if self.options.connection == "local":
@@ -124,17 +126,17 @@ class AdHocCLI(CLI):
         inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
         variable_manager.set_inventory(inventory)
 
-        hosts = inventory.list_hosts(pattern)
         no_hosts = False
-        if len(hosts) == 0:
+        if len(inventory.list_hosts(pattern)) == 0:
+            # Empty inventory
             display.warning("provided hosts list is empty, only localhost is available")
             no_hosts = True
 
-        if self.options.subset:
-            inventory.subset(self.options.subset)
-            if len(inventory.list_hosts(pattern)) == 0 and not no_hosts:
-                # Invalid limit
-                raise AnsibleError("Specified --limit does not match any hosts")
+        inventory.subset(self.options.subset)
+        hosts = inventory.list_hosts(pattern)
+        if len(hosts) == 0 and no_hosts is False:
+            # Invalid limit
+            raise AnsibleError("Specified --limit does not match any hosts")
 
         if self.options.listhosts:
             display.display('  hosts (%d):' % len(hosts))
@@ -158,14 +160,18 @@ class AdHocCLI(CLI):
         play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval)
         play = Play().load(play_ds, variable_manager=variable_manager, loader=loader)
 
-        if self.options.one_line:
+        if self.callback: 
+            cb = self.callback
+        elif self.options.one_line:
             cb = 'oneline'
         else:
             cb = 'minimal'
 
+        run_tree=False
         if self.options.tree:
             C.DEFAULT_CALLBACK_WHITELIST.append('tree')
             C.TREE_DIR = self.options.tree
+            run_tree=True
 
         # now create a task queue manager to execute the play
         self._tqm = None
@@ -177,7 +183,10 @@ class AdHocCLI(CLI):
                     options=self.options,
                     passwords=passwords,
                     stdout_callback=cb,
+                    run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
+                    run_tree=run_tree,
                 )
+
             result = self._tqm.run(play)
         finally:
             if self._tqm:

lib/ansible/cli/console.py

@@ -0,0 +1,444 @@
+# (c) 2014, Nandor Sivok <dominis@haxor.hu>
+# (c) 2016, Redhat Inc
+#
+# ansible-console is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ansible-console is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+########################################################
+# ansible-console is an interactive REPL shell for ansible
+# with built-in tab completion for all the documented modules
+#
+# Available commands:
+#  cd - change host/group (you can use host patterns eg.: app*.dc*:!app01*)
+#  list - list available hosts in the current path
+#  forks - change fork
+#  become - become
+#  ! - forces shell module instead of the ansible module (!yum update -y)
+
+import atexit
+import cmd
+import getpass
+import readline
+import os
+import sys
+
+from ansible import constants as C
+from ansible.cli import CLI
+from ansible.errors import AnsibleError, AnsibleOptionsError
+
+from ansible.executor.task_queue_manager import TaskQueueManager
+from ansible.inventory import Inventory
+from ansible.parsing.dataloader import DataLoader
+from ansible.parsing.splitter import parse_kv
+from ansible.playbook.play import Play
+from ansible.vars import VariableManager
+from ansible.utils import module_docs
+from ansible.utils.color import stringc
+from ansible.utils.unicode import to_unicode, to_str
+from ansible.plugins import module_loader
+
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class ConsoleCLI(CLI, cmd.Cmd):
+
+    modules = []
+
+    def __init__(self, args):
+
+        super(ConsoleCLI, self).__init__(args)
+
+        self.intro = 'Welcome to the ansible console.\nType help or ? to list commands.\n'
+
+        self.groups = []
+        self.hosts = []
+        self.pattern = None
+        self.variable_manager = None
+        self.loader = None
+        self.passwords = dict()
+
+        self.modules = None
+        cmd.Cmd.__init__(self)
+
+    def parse(self):
+        self.parser = CLI.base_parser(
+            usage='%prog <host-pattern> [options]',
+            runas_opts=True,
+            inventory_opts=True,
+            connect_opts=True,
+            check_opts=True,
+            vault_opts=True,
+            fork_opts=True,
+            module_opts=True,
+        )
+
+        # options unique to shell
+        self.parser.add_option('--step', dest='step', action='store_true',
+            help="one-step-at-a-time: confirm each task before running")
+
+        self.parser.set_defaults(cwd='*')
+        self.options, self.args = self.parser.parse_args(self.args[1:])
+
+        display.verbosity = self.options.verbosity
+        self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True)
+
+        return True
+
+    def get_names(self):
+        return dir(self)
+
+    def cmdloop(self):
+        try:
+            cmd.Cmd.cmdloop(self)
+        except KeyboardInterrupt:
+            self.do_exit(self)
+
+    def set_prompt(self):
+        login_user = self.options.remote_user or getpass.getuser()
+        self.selected = self.inventory.list_hosts(self.options.cwd)
+        prompt = "%s@%s (%d)[f:%s]" % (login_user, self.options.cwd, len(self.selected), self.options.forks)
+        if self.options.become and self.options.become_user in [None, 'root']:
+            prompt += "# "
+            color = C.COLOR_ERROR
+        else:
+            prompt += "$ "
+            color = C.COLOR_HIGHLIGHT
+        self.prompt = stringc(prompt, color)
+
+    def list_modules(self):
+        modules = set()
+        if self.options.module_path is not None:
+            for i in self.options.module_path.split(os.pathsep):
+                module_loader.add_directory(i)
+
+        module_paths = module_loader._get_paths()
+        for path in module_paths:
+            if path is not None:
+                modules.update(self._find_modules_in_path(path))
+        return modules
+
+    def _find_modules_in_path(self, path):
+
+        if os.path.isdir(path):
+            for module in os.listdir(path):
+                if module.startswith('.'):
+                    continue
+                elif os.path.isdir(module):
+                    self._find_modules_in_path(module)
+                elif module.startswith('__'):
+                    continue
+                elif any(module.endswith(x) for x in C.BLACKLIST_EXTS):
+                    continue
+                elif module in C.IGNORE_FILES:
+                    continue
+                elif module.startswith('_'):
+                    fullpath = '/'.join([path,module])
+                    if os.path.islink(fullpath): # avoids aliases
+                        continue
+                    module = module.replace('_', '', 1)
+
+                module = os.path.splitext(module)[0] # removes the extension
+                yield module
+
+    def default(self, arg, forceshell=False):
+        """ actually runs modules """
+        if arg.startswith("#"):
+            return False
+
+        if not self.options.cwd:
+            display.error("No host found")
+            return False
+
+        if arg.split()[0] in self.modules:
+            module = arg.split()[0]
+            module_args = ' '.join(arg.split()[1:])
+        else:
+            module = 'shell'
+            module_args = arg
+
+        if forceshell is True:
+            module = 'shell'
+            module_args = arg
+
+        self.options.module_name = module
+
+        result = None
+        try:
+            check_raw = self.options.module_name in ('command', 'shell', 'script', 'raw')
+            play_ds = dict(
+                name = "Ansible Shell",
+                hosts = self.options.cwd,
+                gather_facts = 'no',
+                tasks = [ dict(action=dict(module=module, args=parse_kv(module_args, check_raw=check_raw)))]
+            )
+            play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader)
+        except Exception as e:
+            display.error(u"Unable to build command: %s" % to_unicode(e))
+            return False
+
+        try:
+            cb = 'minimal' #FIXME: make callbacks configurable
+            # now create a task queue manager to execute the play
+            self._tqm = None
+            try:
+                self._tqm = TaskQueueManager(
+                        inventory=self.inventory,
+                        variable_manager=self.variable_manager,
+                        loader=self.loader,
+                        options=self.options,
+                        passwords=self.passwords,
+                        stdout_callback=cb,
+                        run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
+                        run_tree=False,
+                    )
+
+                result = self._tqm.run(play)
+            finally:
+                if self._tqm:
+                    self._tqm.cleanup()
+
+            if result is None:
+                display.error("No hosts found")
+                return False
+        except KeyboardInterrupt:
+            display.error('User interrupted execution')
+            return False
+        except Exception as e:
+            display.error(to_unicode(e))
+            #FIXME: add traceback in very very verbose mode
+            return False
+
+    def emptyline(self):
+        return
+
+    def do_shell(self, arg):
+        """
+        You can run shell commands through the shell module.
+
+        eg.:
+        shell ps uax | grep java | wc -l
+        shell killall python
+        shell halt -n
+
+        You can use the ! to force the shell module. eg.:
+        !ps aux | grep java | wc -l
+        """
+        self.default(arg, True)
+
+    def do_forks(self, arg):
+        """Set the number of forks"""
+        if not arg:
+            display.display('Usage: forks <number>')
+            return
+        self.options.forks = int(arg)
+        self.set_prompt()
+
+    do_serial = do_forks
+
+    def do_verbosity(self, arg):
+        """Set verbosity level"""
+        if not arg:
+            display.display('Usage: verbosity <number>')
+        else:
+            display.verbosity = int(arg)
+            display.v('verbosity level set to %s' % arg)
+
+    def do_cd(self, arg):
+        """
+            Change active host/group. You can use hosts patterns as well eg.:
+            cd webservers
+            cd webservers:dbservers
+            cd webservers:!phoenix
+            cd webservers:&staging
+            cd webservers:dbservers:&staging:!phoenix
+        """
+        if not arg:
+            self.options.cwd = '*'
+        elif arg == '..':
+            try:
+                self.options.cwd = self.inventory.groups_for_host(self.options.cwd)[1].name
+            except Exception:
+                self.options.cwd = ''
+        elif arg in '/*':
+            self.options.cwd = 'all'
+        elif self.inventory.get_hosts(arg):
+            self.options.cwd = arg
+        else:
+            display.display("no host matched")
+
+        self.set_prompt()
+
+    def do_list(self, arg):
+        """List the hosts in the current group"""
+        if arg == 'groups':
+            for group in self.groups:
+                display.display(group)
+        else:
+            for host in self.selected:
+                display.display(host.name)
+
+    def do_become(self, arg):
+        """Toggle whether plays run with become"""
+        if arg:
+            self.options.become_user = arg
+            display.v("become changed to %s" % self.options.become)
+            self.set_prompt()
+        else:
+            display.display("Please specify become value, e.g. `become yes`")
+
+    def do_remote_user(self, arg):
+        """Given a username, set the remote user plays are run by"""
+        if arg:
+            self.options.remote_user = arg
+            self.set_prompt()
+        else:
+            display.display("Please specify a remote user, e.g. `remote_user root`")
+
+    def do_become_user(self, arg):
+        """Given a username, set the user that plays are run by when using become"""
+        if arg:
+            self.options.become_user = arg
+        else:
+            display.display("Please specify a user, e.g. `become_user jenkins`")
+            display.v("Current user is %s" % self.options.become_user)
+        self.set_prompt()
+
+    def do_become_method(self, arg):
+        """Given a become_method, set the privilege escalation method when using become"""
+        if arg:
+            self.options.become_method = arg
+            display.v("become_method changed to %s" % self.options.become_method)
+        else:
+            display.display("Please specify a become_method, e.g. `become_method su`")
+
+    def do_exit(self, args):
+        """Exits from the console"""
+        sys.stdout.write('\n')
+        return -1
+
+    do_EOF = do_exit
+
+    def helpdefault(self, module_name):
+        if module_name in self.modules:
+            in_path = module_loader.find_plugin(module_name)
+            if in_path:
+                oc, a, _ = module_docs.get_docstring(in_path)
+                if oc:
+                    display.display(oc['short_description'])
+                    display.display('Parameters:')
+                    for opt in oc['options'].keys():
+                        display.display('  ' + stringc(opt, C.COLOR_HIGHLIGHT) + ' ' + oc['options'][opt]['description'][0])
+                else:
+                    display.error('No documentation found for %s.' % module_name)
+            else:
+                display.error('%s is not a valid command, use ? to list all valid commands.' % module_name)
+
+    def complete_cd(self, text, line, begidx, endidx):
+        mline = line.partition(' ')[2]
+        offs = len(mline) - len(text)
+
+        if self.options.cwd in ('all','*','\\'):
+            completions = self.hosts + self.groups
+        else:
+            completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)]
+
+        return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))]
+
+    def completedefault(self, text, line, begidx, endidx):
+        if line.split()[0] in self.modules:
+            mline = line.split(' ')[-1]
+            offs = len(mline) - len(text)
+            completions = self.module_args(line.split()[0])
+
+            return [s[offs:] + '=' for s in completions if s.startswith(mline)]
+
+    def module_args(self, module_name):
+        in_path = module_loader.find_plugin(module_name)
+        oc, a, _ = module_docs.get_docstring(in_path)
+        return oc['options'].keys()
+
+
+    def run(self):
+
+        super(ConsoleCLI, self).run()
+
+        sshpass    = None
+        becomepass = None
+        vault_pass = None
+
+        # hosts
+        if len(self.args) != 1:
+            self.pattern = 'all'
+        else:
+            self.pattern = self.args[0]
+        self.options.cwd = self.pattern
+
+
+        # dynamically add modules as commands
+        self.modules = self.list_modules()
+        for module in self.modules:
+            setattr(self, 'do_' + module, lambda arg, module=module: self.default(module + ' ' + arg))
+            setattr(self, 'help_' + module, lambda module=module: self.helpdefault(module))
+
+        self.normalize_become_options()
+        (sshpass, becomepass) = self.ask_passwords()
+        self.passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }
+
+        self.loader = DataLoader()
+
+        if self.options.vault_password_file:
+            # read vault_pass from a file
+            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=self.loader)
+            self.loader.set_vault_password(vault_pass)
+        elif self.options.ask_vault_pass:
+            vault_pass = self.ask_vault_passwords()[0]
+            self.loader.set_vault_password(vault_pass)
+
+        self.variable_manager = VariableManager()
+        self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=self.options.inventory)
+        self.variable_manager.set_inventory(self.inventory)
+
+        if len(self.inventory.list_hosts(self.pattern)) == 0:
+            # Empty inventory
+            display.warning("provided hosts list is empty, only localhost is available")
+
+        self.inventory.subset(self.options.subset)
+        self.groups = self.inventory.list_groups()
+        self.hosts = [x.name for x in self.inventory.list_hosts(self.pattern)]
+
+        # This hack is to work around readline issues on a mac:
+        #  http://stackoverflow.com/a/7116997/541202
+        if 'libedit' in readline.__doc__:
+            readline.parse_and_bind("bind ^I rl_complete")
+        else:
+            readline.parse_and_bind("tab: complete")
+
+        histfile = os.path.join(os.path.expanduser("~"), ".ansible-console_history")
+        try:
+            readline.read_history_file(histfile)
+        except IOError:
+            pass
+
+        atexit.register(readline.write_history_file, histfile)
+        self.set_prompt()
+        self.cmdloop()
+

lib/ansible/cli/doc.py

@@ -26,6 +26,7 @@ import textwrap
 
 from ansible.compat.six import iteritems
 
+from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.plugins import module_loader
 from ansible.cli import CLI
@@ -41,9 +42,6 @@ except ImportError:
 class DocCLI(CLI):
     """ Vault command line class """
 
-    BLACKLIST_EXTS = ('.pyc', '.swp', '.bak', '~', '.rpm', '.md', '.txt')
-    IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README", "VERSION", "GUIDELINES", "test-docs.sh"]
-
     def __init__(self, args):
 
         super(DocCLI, self).__init__(args)
@@ -62,7 +60,7 @@ class DocCLI(CLI):
         self.parser.add_option("-s", "--snippet", action="store_true", default=False, dest='show_snippet',
                 help='Show playbook snippet for specified module(s)')
 
-        self.options, self.args = self.parser.parse_args()
+        self.options, self.args = self.parser.parse_args(self.args[1:])
         display.verbosity = self.options.verbosity
 
     def run(self):
@@ -90,12 +88,13 @@ class DocCLI(CLI):
         for module in self.args:
 
             try:
-                filename = module_loader.find_plugin(module)
+                # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
+                filename = module_loader.find_plugin(module, mod_type='.py')
                 if filename is None:
                     display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader)))
                     continue
 
-                if any(filename.endswith(x) for x in self.BLACKLIST_EXTS):
+                if any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
                     continue
 
                 try:
@@ -142,11 +141,11 @@ class DocCLI(CLI):
                     continue
                 elif os.path.isdir(module):
                     self.find_modules(module)
-                elif any(module.endswith(x) for x in self.BLACKLIST_EXTS):
+                elif any(module.endswith(x) for x in C.BLACKLIST_EXTS):
                     continue
                 elif module.startswith('__'):
                     continue
-                elif module in self.IGNORE_FILES:
+                elif module in C.IGNORE_FILES:
                     continue
                 elif module.startswith('_'):
                     fullpath = '/'.join([path,module])
@@ -167,7 +166,8 @@ class DocCLI(CLI):
             if module in module_docs.BLACKLIST_MODULES:
                 continue
 
-            filename = module_loader.find_plugin(module)
+            # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
+            filename = module_loader.find_plugin(module, mod_type='.py')
 
             if filename is None:
                 continue

lib/ansible/cli/galaxy.py

@@ -22,10 +22,10 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os
 import os.path
 import sys
 import yaml
+import time
 
 from collections import defaultdict
 from jinja2 import Environment
@@ -36,7 +36,10 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.galaxy import Galaxy
 from ansible.galaxy.api import GalaxyAPI
 from ansible.galaxy.role import GalaxyRole
+from ansible.galaxy.login import GalaxyLogin
+from ansible.galaxy.token import GalaxyToken
 from ansible.playbook.role.requirement import RoleRequirement
+from ansible.utils.unicode import to_unicode
 
 try:
     from __main__ import display
@@ -44,14 +47,12 @@ except ImportError:
     from ansible.utils.display import Display
     display = Display()
 
-
 class GalaxyCLI(CLI):
 
-    VALID_ACTIONS = ("init", "info", "install", "list", "remove", "search")
     SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" )
-
+    VALID_ACTIONS = ("delete", "import", "info", "init", "install", "list", "login", "remove", "search", "setup")
+    
     def __init__(self, args):
-
         self.api = None
         self.galaxy = None
         super(GalaxyCLI, self).__init__(args)
@@ -67,7 +68,17 @@ class GalaxyCLI(CLI):
         self.set_action()
 
         # options specific to actions
-        if self.action == "info":
+        if self.action == "delete":
+            self.parser.set_usage("usage: %prog delete [options] github_user github_repo")
+        elif self.action == "import":
+            self.parser.set_usage("usage: %prog import [options] github_user github_repo")
+            self.parser.add_option('--no-wait', dest='wait', action='store_false', default=True,
+                help='Don\'t wait for import results.')
+            self.parser.add_option('--branch', dest='reference',
+                help='The name of a branch to import. Defaults to the repository\'s default branch (usually master)')
+            self.parser.add_option('--status', dest='check_status', action='store_true', default=False,
+                help='Check the status of the most recent import request for given github_user/github_repo.')
+        elif self.action == "info":
             self.parser.set_usage("usage: %prog info [options] role_name[,version]")
         elif self.action == "init":
             self.parser.set_usage("usage: %prog init [options] role_name")
@@ -88,31 +99,42 @@ class GalaxyCLI(CLI):
             self.parser.set_usage("usage: %prog remove role1 role2 ...")
         elif self.action == "list":
             self.parser.set_usage("usage: %prog list [role_name]")
+        elif self.action == "login":
+            self.parser.set_usage("usage: %prog login [options]")
+            self.parser.add_option('--github-token', dest='token', default=None,
+                help='Identify with github token rather than username and password.')
         elif self.action == "search":
             self.parser.add_option('--platforms', dest='platforms',
                 help='list of OS platforms to filter by')
             self.parser.add_option('--galaxy-tags', dest='tags',
                 help='list of galaxy tags to filter by')
-            self.parser.set_usage("usage: %prog search [<search_term>] [--galaxy-tags <galaxy_tag1,galaxy_tag2>] [--platforms platform]")
+            self.parser.add_option('--author', dest='author',
+                help='GitHub username')
+            self.parser.set_usage("usage: %prog search [searchterm1 searchterm2] [--galaxy-tags galaxy_tag1,galaxy_tag2] [--platforms platform1,platform2] [--author username]")
+        elif self.action == "setup":
+            self.parser.set_usage("usage: %prog setup [options] source github_user github_repo secret")
+            self.parser.add_option('--remove', dest='remove_id', default=None,
+                help='Remove the integration matching the provided ID value. Use --list to see ID values.')
+            self.parser.add_option('--list', dest="setup_list", action='store_true', default=False,
+                help='List all of your integrations.')
 
         # options that apply to more than one action
-        if self.action != "init":
+        if not self.action in ("delete","import","init","login","setup"):
             self.parser.add_option('-p', '--roles-path', dest='roles_path', default=C.DEFAULT_ROLES_PATH,
                 help='The path to the directory containing your roles. '
                      'The default is the roles_path configured in your '
                      'ansible.cfg file (/etc/ansible/roles if not configured)')
 
-        if self.action in ("info","init","install","search"):
-            self.parser.add_option('-s', '--server', dest='api_server', default="https://galaxy.ansible.com",
+        if self.action in ("import","info","init","install","login","search","setup","delete"):
+            self.parser.add_option('-s', '--server', dest='api_server', default=C.GALAXY_SERVER,
                 help='The API server destination')
-            self.parser.add_option('-c', '--ignore-certs', action='store_false', dest='validate_certs', default=True,
+            self.parser.add_option('-c', '--ignore-certs', action='store_true', dest='ignore_certs', default=False,
                 help='Ignore SSL certificate validation errors.')
 
         if self.action in ("init","install"):
             self.parser.add_option('-f', '--force', dest='force', action='store_true', default=False,
                 help='Force overwriting an existing role')
 
-        # get options, args and galaxy object
         self.options, self.args =self.parser.parse_args()
         display.verbosity = self.options.verbosity
         self.galaxy = Galaxy(self.options)
@@ -120,15 +142,13 @@ class GalaxyCLI(CLI):
         return True
 
     def run(self):
-
+        
         super(GalaxyCLI, self).run()
 
         # if not offline, get connect to galaxy api
-        if self.action in ("info","install", "search") or (self.action == 'init' and not self.options.offline):
-            api_server = self.options.api_server
-            self.api = GalaxyAPI(self.galaxy, api_server)
-            if not self.api:
-                raise AnsibleError("The API server (%s) is not responding, please try again later." % api_server)
+        if self.action in ("import","info","install","search","login","setup","delete") or \
+            (self.action == 'init' and not self.options.offline):
+            self.api = GalaxyAPI(self.galaxy)
 
         self.execute()
 
@@ -142,8 +162,8 @@ class GalaxyCLI(CLI):
 
     def _display_role_info(self, role_info):
 
-        text = "\nRole: %s \n" % role_info['name']
-        text += "\tdescription: %s \n" % role_info.get('description', '')
+        text = [u"", u"Role: %s" % to_unicode(role_info['name'])]
+        text.append(u"\tdescription: %s" % role_info.get('description', ''))
 
         for k in sorted(role_info.keys()):
 
@@ -152,14 +172,15 @@ class GalaxyCLI(CLI):
 
             if isinstance(role_info[k], dict):
                 text += "\t%s: \n" % (k)
+                text.append(u"\t%s:" % (k))
                 for key in sorted(role_info[k].keys()):
                     if key in self.SKIP_INFO_KEYS:
                         continue
-                    text += "\t\t%s: %s\n" % (key, role_info[k][key])
+                    text.append(u"\t\t%s: %s" % (key, role_info[k][key]))
             else:
-                text += "\t%s: %s\n" % (k, role_info[k])
+                text.append(u"\t%s: %s" % (k, role_info[k]))
 
-        return text
+        return u'\n'.join(text)
 
 ############################
 # execute actions
@@ -188,7 +209,7 @@ class GalaxyCLI(CLI):
                             "however it will reset any main.yml files that may have\n"
                             "been modified there already." % role_path)
 
-        # create the default README.md
+        # create default README.md
         if not os.path.exists(role_path):
             os.makedirs(role_path)
         readme_path = os.path.join(role_path, "README.md")
@@ -196,9 +217,16 @@ class GalaxyCLI(CLI):
         f.write(self.galaxy.default_readme)
         f.close()
 
+        # create default .travis.yml
+        travis = Environment().from_string(self.galaxy.default_travis).render()
+        f = open(os.path.join(role_path, '.travis.yml'), 'w')
+        f.write(travis)
+        f.close()
+
         for dir in GalaxyRole.ROLE_DIRS:
             dir_path = os.path.join(init_path, role_name, dir)
             main_yml_path = os.path.join(dir_path, 'main.yml')
+
             # create the directory if it doesn't exist already
             if not os.path.exists(dir_path):
                 os.makedirs(dir_path)
@@ -223,6 +251,7 @@ class GalaxyCLI(CLI):
 
                 inject = dict(
                     author = 'your name',
+                    description = 'your description',
                     company = 'your company (optional)',
                     license = 'license (GPLv2, CC-BY, etc)',
                     issue_tracker_url = 'http://example.com/issue/tracker',
@@ -234,6 +263,20 @@ class GalaxyCLI(CLI):
                 f.write(rendered_meta)
                 f.close()
                 pass
+            elif dir == "tests":
+                # create tests/test.yml
+                inject = dict(
+                    role_name = role_name
+                )
+                playbook = Environment().from_string(self.galaxy.default_test).render(inject)
+                f = open(os.path.join(dir_path, 'test.yml'), 'w')
+                f.write(playbook)
+                f.close()
+
+                # create tests/inventory
+                f = open(os.path.join(dir_path, 'inventory'), 'w')
+                f.write('localhost')
+                f.close()
             elif dir not in ('files','templates'):
                 # just write a (mostly) empty YAML file for main.yml
                 f = open(main_yml_path, 'w')
@@ -282,9 +325,11 @@ class GalaxyCLI(CLI):
             if role_spec:
                 role_info.update(role_spec)
 
-            data += self._display_role_info(role_info)
+            data = self._display_role_info(role_info)
+            ### FIXME: This is broken in both 1.9 and 2.0 as
+            # _display_role_info() always returns something
             if not data:
-                data += "\n- the role %s was not found" % role
+                data = u"\n- the role %s was not found" % role
 
         self.pager(data)
 
@@ -325,7 +370,7 @@ class GalaxyCLI(CLI):
 
                     for role in required_roles:
                         role = RoleRequirement.role_yaml_parse(role)
-                        display.debug('found role %s in yaml file' % str(role))
+                        display.vvv('found role %s in yaml file' % str(role))
                         if 'name' not in role and 'scm' not in role:
                             raise AnsibleError("Must specify name or src for role")
                         roles_left.append(GalaxyRole(self.galaxy, **role))
@@ -345,10 +390,11 @@ class GalaxyCLI(CLI):
             # roles were specified directly, so we'll just go out grab them
             # (and their dependencies, unless the user doesn't want us to).
             for rname in self.args:
-                roles_left.append(GalaxyRole(self.galaxy, rname.strip()))
+                role = RoleRequirement.role_yaml_parse(rname.strip())
+                roles_left.append(GalaxyRole(self.galaxy, **role))
 
         for role in roles_left:
-            display.debug('Installing role %s ' % role.name)
+            display.vvv('Installing role %s ' % role.name)
             # query the galaxy API for the role data
 
             if role.install_info is not None and not force:
@@ -458,21 +504,187 @@ class GalaxyCLI(CLI):
         return 0
 
     def execute_search(self):
-
+        page_size = 1000
         search = None
-        if len(self.args) > 1:
-            raise AnsibleOptionsError("At most a single search term is allowed.")
-        elif len(self.args) == 1:
-            search = self.args.pop()
 
-        response = self.api.search_roles(search, self.options.platforms, self.options.tags)
+        if len(self.args):
+            terms = []
+            for i in range(len(self.args)):
+               terms.append(self.args.pop())
+            search = '+'.join(terms[::-1])
 
-        if 'count' in response:
-            display.display("Found %d roles matching your search:\n" % response['count'])
+        if not search and not self.options.platforms and not self.options.tags and not self.options.author:
+            raise AnsibleError("Invalid query. At least one search term, platform, galaxy tag or author must be provided.")
 
-        data = ''
-        if 'results' in response:
-            for role in response['results']:
-                data += self._display_role_info(role)
+        response = self.api.search_roles(search, platforms=self.options.platforms,
+            tags=self.options.tags, author=self.options.author, page_size=page_size)
 
+        if response['count'] == 0:
+            display.display("No roles match your search.", color=C.COLOR_ERROR)
+            return True
+
+        data = [u'']
+
+        if response['count'] > page_size:
+            data.append(u"Found %d roles matching your search. Showing first %s." % (response['count'], page_size))
+        else:
+            data.append(u"Found %d roles matching your search:" % response['count'])
+
+        max_len = []
+        for role in response['results']:
+            max_len.append(len(role['username'] + '.' + role['name']))
+        name_len = max(max_len)
+        format_str = u" %%-%ds %%s" % name_len
+        data.append(u'')
+        data.append(format_str % (u"Name", u"Description"))
+        data.append(format_str % (u"----", u"-----------"))
+        for role in response['results']:
+            data.append(format_str % (u'%s.%s' % (role['username'], role['name']), role['description']))
+
+        data = u'\n'.join(data)
         self.pager(data)
+
+        return True
+
+    def execute_login(self):
+        """
+        Verify user's identify via Github and retreive an auth token from Galaxy.
+        """
+        # Authenticate with github and retrieve a token
+        if self.options.token is None:
+            login = GalaxyLogin(self.galaxy)
+            github_token = login.create_github_token()
+        else:
+            github_token = self.options.token
+
+        galaxy_response = self.api.authenticate(github_token)
+
+        if self.options.token is None:
+            # Remove the token we created
+            login.remove_github_token()
+
+        # Store the Galaxy token
+        token = GalaxyToken()
+        token.set(galaxy_response['token'])
+
+        display.display("Succesfully logged into Galaxy as %s" % galaxy_response['username'])
+        return 0
+
+    def execute_import(self):
+        """
+        Import a role into Galaxy
+        """
+
+        colors = {
+            'INFO':    'normal',
+            'WARNING': C.COLOR_WARN,
+            'ERROR':   C.COLOR_ERROR,
+            'SUCCESS': C.COLOR_OK,
+            'FAILED': C.COLOR_ERROR,
+        }
+
+        if len(self.args) < 2:
+            raise AnsibleError("Expected a github_username and github_repository. Use --help.")
+
+        github_repo = self.args.pop()
+        github_user = self.args.pop()
+
+        if self.options.check_status:
+            task = self.api.get_import_task(github_user=github_user, github_repo=github_repo)
+        else:
+            # Submit an import request
+            task = self.api.create_import_task(github_user, github_repo, reference=self.options.reference)
+
+            if len(task) > 1:
+                # found multiple roles associated with github_user/github_repo
+                display.display("WARNING: More than one Galaxy role associated with Github repo %s/%s." % (github_user,github_repo),
+                    color='yellow')
+                display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED)
+                for t in task:
+                    display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED)
+                display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED)
+                return 0
+            # found a single role as expected
+            display.display("Successfully submitted import request %d" % task[0]['id'])
+            if not self.options.wait:
+                display.display("Role name: %s" % task[0]['summary_fields']['role']['name'])
+                display.display("Repo: %s/%s" % (task[0]['github_user'],task[0]['github_repo']))
+
+        if self.options.check_status or self.options.wait:
+            # Get the status of the import
+            msg_list = []
+            finished = False
+            while not finished:
+                task = self.api.get_import_task(task_id=task[0]['id'])
+                for msg in task[0]['summary_fields']['task_messages']:
+                    if msg['id'] not in msg_list:
+                        display.display(msg['message_text'], color=colors[msg['message_type']])
+                        msg_list.append(msg['id'])
+                if task[0]['state'] in ['SUCCESS', 'FAILED']:
+                    finished = True
+                else:
+                    time.sleep(10)
+
+        return 0
+
+    def execute_setup(self):
+        """
+        Setup an integration from Github or Travis
+        """
+
+        if self.options.setup_list:
+            # List existing integration secrets
+            secrets = self.api.list_secrets()
+            if len(secrets) == 0:
+                # None found
+                display.display("No integrations found.")
+                return 0
+            display.display(u'\n' + "ID         Source     Repo", color=C.COLOR_OK)
+            display.display("---------- ---------- ----------", color=C.COLOR_OK)
+            for secret in secrets:
+                display.display("%-10s %-10s %s/%s" % (secret['id'], secret['source'], secret['github_user'],
+                    secret['github_repo']),color=C.COLOR_OK)
+            return 0
+
+        if self.options.remove_id:
+            # Remove a secret
+            self.api.remove_secret(self.options.remove_id)
+            display.display("Secret removed. Integrations using this secret will not longer work.", color=C.COLOR_OK)
+            return 0
+
+        if len(self.args) < 4:
+            raise AnsibleError("Missing one or more arguments. Expecting: source github_user github_repo secret")
+            return 0
+
+        secret = self.args.pop()
+        github_repo = self.args.pop()
+        github_user = self.args.pop()
+        source = self.args.pop()
+
+        resp = self.api.add_secret(source, github_user, github_repo, secret)
+        display.display("Added integration for %s %s/%s" % (resp['source'], resp['github_user'], resp['github_repo']))
+
+        return 0
+
+    def execute_delete(self):
+        """
+        Delete a role from galaxy.ansible.com
+        """
+
+        if len(self.args) < 2:
+            raise AnsibleError("Missing one or more arguments. Expected: github_user github_repo")
+
+        github_repo = self.args.pop()
+        github_user = self.args.pop()
+        resp = self.api.delete_role(github_user, github_repo)
+
+        if len(resp['deleted_roles']) > 1:
+            display.display("Deleted the following roles:")
+            display.display("ID     User            Name")
+            display.display("------ --------------- ----------")
+            for role in resp['deleted_roles']:
+                display.display("%-8s %-15s %s" % (role.id,role.namespace,role.name))
+
+        display.display(resp['status'])
+
+        return True

lib/ansible/cli/playbook.py

@@ -30,6 +30,8 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.playbook_executor import PlaybookExecutor
 from ansible.inventory import Inventory
 from ansible.parsing.dataloader import DataLoader
+from ansible.playbook.block import Block
+from ansible.playbook.play_context import PlayContext
 from ansible.utils.vars import load_extra_vars
 from ansible.vars import VariableManager
 
@@ -72,7 +74,7 @@ class PlaybookCLI(CLI):
         parser.add_option('--start-at-task', dest='start_at_task',
             help="start the playbook at the task matching this name")
 
-        self.options, self.args = parser.parse_args()
+        self.options, self.args = parser.parse_args(self.args[1:])
 
 
         self.parser = parser
@@ -152,18 +154,10 @@ class PlaybookCLI(CLI):
             for p in results:
 
                 display.display('\nplaybook: %s' % p['playbook'])
-                i = 1
-                for play in p['plays']:
-                    if play.name:
-                        playname = play.name
-                    else:
-                        playname = '#' + str(i)
-
-                    msg = "\n  PLAY: %s" % (playname)
-                    mytags = set()
-                    if self.options.listtags and play.tags:
-                        mytags = mytags.union(set(play.tags))
-                        msg += '    TAGS: [%s]' % (','.join(mytags))
+                for idx, play in enumerate(p['plays']):
+                    msg = "\n  play #%d (%s): %s" % (idx + 1, ','.join(play.hosts), play.name)
+                    mytags = set(play.tags)
+                    msg += '\tTAGS: [%s]' % (','.join(mytags))
 
                     if self.options.listhosts:
                         playhosts = set(inventory.get_hosts(play.hosts))
@@ -173,23 +167,48 @@ class PlaybookCLI(CLI):
 
                     display.display(msg)
 
+                    all_tags = set()
                     if self.options.listtags or self.options.listtasks:
-                        taskmsg = '    tasks:'
+                        taskmsg = ''
+                        if self.options.listtasks:
+                            taskmsg = '    tasks:\n'
 
+                        def _process_block(b):
+                            taskmsg = ''
+                            for task in b.block:
+                                if isinstance(task, Block):
+                                    taskmsg += _process_block(task)
+                                else:
+                                    if task.action == 'meta':
+                                        continue
+
+                                    all_tags.update(task.tags)
+                                    if self.options.listtasks:
+                                        cur_tags = list(mytags.union(set(task.tags)))
+                                        cur_tags.sort()
+                                        if task.name:
+                                            taskmsg += "      %s" % task.get_name()
+                                        else:
+                                            taskmsg += "      %s" % task.action
+                                        taskmsg += "\tTAGS: [%s]\n" % ', '.join(cur_tags)
+
+                            return taskmsg
+
+                        all_vars = variable_manager.get_vars(loader=loader, play=play)
+                        play_context = PlayContext(play=play, options=self.options)
                         for block in play.compile():
+                            block = block.filter_tagged_tasks(play_context, all_vars)
                             if not block.has_tasks():
                                 continue
+                            taskmsg += _process_block(block)
 
-                            j = 1
-                            for task in block.block:
-                                taskmsg += "\n      %s" % task
-                                if self.options.listtags and task.tags:
-                                    taskmsg += "    TAGS: [%s]" % ','.join(mytags.union(set(task.tags)))
-                                j = j + 1
+                        if self.options.listtags:
+                            cur_tags = list(mytags.union(all_tags))
+                            cur_tags.sort()
+                            taskmsg += "      TASK TAGS: [%s]\n" % ', '.join(cur_tags)
 
                         display.display(taskmsg)
 
-                    i = i + 1
             return 0
         else:
             return results

lib/ansible/cli/pull.py

@@ -64,18 +64,24 @@ class PullCLI(CLI):
             subset_opts=True,
             inventory_opts=True,
             module_opts=True,
+            runas_prompt_opts=True,
         )
 
         # options unique to pull
-        self.parser.add_option('--purge', default=False, action='store_true', help='purge checkout after playbook run')
+        self.parser.add_option('--purge', default=False, action='store_true',
+            help='purge checkout after playbook run')
         self.parser.add_option('-o', '--only-if-changed', dest='ifchanged', default=False, action='store_true',
             help='only run the playbook if the repository has been updated')
         self.parser.add_option('-s', '--sleep', dest='sleep', default=None,
             help='sleep for random interval (between 0 and n number of seconds) before starting. This is a useful way to disperse git requests')
         self.parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
             help='run the playbook even if the repository could not be updated')
-        self.parser.add_option('-d', '--directory', dest='dest', default='~/.ansible/pull', help='directory to checkout repository to')
-        self.parser.add_option('-U', '--url', dest='url', default=None, help='URL of the playbook repository')
+        self.parser.add_option('-d', '--directory', dest='dest', default=None,
+            help='directory to checkout repository to')
+        self.parser.add_option('-U', '--url', dest='url', default=None,
+            help='URL of the playbook repository')
+        self.parser.add_option('--full', dest='fullclone', action='store_true',
+            help='Do a full clone, instead of a shallow one.')
         self.parser.add_option('-C', '--checkout', dest='checkout',
             help='branch/tag/commit to checkout.  ' 'Defaults to behavior of repository module.')
         self.parser.add_option('--accept-host-key', default=False, dest='accept_host_key', action='store_true',
@@ -86,7 +92,16 @@ class PullCLI(CLI):
             help='verify GPG signature of checked out commit, if it fails abort running the playbook.'
                  ' This needs the corresponding VCS module to support such an operation')
 
-        self.options, self.args = self.parser.parse_args()
+        # for pull we don't wan't a default
+        self.parser.set_defaults(inventory=None)
+
+        self.options, self.args = self.parser.parse_args(self.args[1:])
+
+        if not self.options.dest:
+            hostname = socket.getfqdn()
+            # use a hostname dependent directory, in case of $HOME on nfs
+            self.options.dest = os.path.join('~/.ansible/pull', hostname)
+        self.options.dest = os.path.expandvars(os.path.expanduser(self.options.dest))
 
         if self.options.sleep:
             try:
@@ -119,18 +134,18 @@ class PullCLI(CLI):
         node = platform.node()
         host = socket.getfqdn()
         limit_opts = 'localhost,%s,127.0.0.1' % ','.join(set([host, node, host.split('.')[0], node.split('.')[0]]))
-        base_opts = '-c local "%s"' % limit_opts
+        base_opts = '-c local '
         if self.options.verbosity > 0:
             base_opts += ' -%s' % ''.join([ "v" for x in range(0, self.options.verbosity) ])
 
         # Attempt to use the inventory passed in as an argument
-        # It might not yet have been downloaded so use localhost if note
-        if not self.options.inventory or not os.path.exists(self.options.inventory):
+        # It might not yet have been downloaded so use localhost as default
+        if not self.options.inventory or ( ',' not in self.options.inventory and not os.path.exists(self.options.inventory)):
             inv_opts = 'localhost,'
         else:
             inv_opts = self.options.inventory
 
-        #TODO: enable more repo modules hg/svn?
+        #FIXME: enable more repo modules hg/svn?
         if self.options.module_name == 'git':
             repo_opts = "name=%s dest=%s" % (self.options.url, self.options.dest)
             if self.options.checkout:
@@ -145,14 +160,16 @@ class PullCLI(CLI):
             if self.options.verify:
                 repo_opts += ' verify_commit=yes'
 
+            if not self.options.fullclone:
+                repo_opts += ' depth=1'
+
         path = module_loader.find_plugin(self.options.module_name)
         if path is None:
             raise AnsibleOptionsError(("module '%s' not found.\n" % self.options.module_name))
 
         bin_path = os.path.dirname(os.path.abspath(sys.argv[0]))
-        cmd = '%s/ansible -i "%s" %s -m %s -a "%s"' % (
-            bin_path, inv_opts, base_opts, self.options.module_name, repo_opts
-        )
+        # hardcode local and inventory/host as this is just meant to fetch the repo
+        cmd = '%s/ansible -i "%s" %s -m %s -a "%s" all -l "%s"' % (bin_path, inv_opts, base_opts, self.options.module_name, repo_opts, limit_opts)
 
         for ev in self.options.extra_vars:
             cmd += ' -e "%s"' % ev
@@ -163,6 +180,8 @@ class PullCLI(CLI):
             time.sleep(self.options.sleep)
 
         # RUN the Checkout command
+        display.debug("running ansible with VCS module to checkout repo")
+        display.vvvv('EXEC: %s' % cmd)
         rc, out, err = run_cmd(cmd, live=True)
 
         if rc != 0:
@@ -174,8 +193,7 @@ class PullCLI(CLI):
             display.display("Repository has not changed, quitting.")
             return 0
 
-        playbook = self.select_playbook(path)
-
+        playbook = self.select_playbook(self.options.dest)
         if playbook is None:
             raise AnsibleOptionsError("Could not find a playbook to run.")
 
@@ -187,16 +205,20 @@ class PullCLI(CLI):
             cmd += ' -i "%s"' % self.options.inventory
         for ev in self.options.extra_vars:
             cmd += ' -e "%s"' % ev
-        if self.options.ask_sudo_pass:
-            cmd += ' -K'
+        if self.options.ask_sudo_pass or self.options.ask_su_pass or self.options.become_ask_pass:
+            cmd += ' --ask-become-pass'
         if self.options.tags:
             cmd += ' -t "%s"' % self.options.tags
-        if self.options.limit:
-            cmd += ' -l "%s"' % self.options.limit
+        if self.options.subset:
+            cmd += ' -l "%s"' % self.options.subset
+        else:
+            cmd += ' -l "%s"' % limit_opts
 
         os.chdir(self.options.dest)
 
         # RUN THE PLAYBOOK COMMAND
+        display.debug("running ansible-playbook to do actual work")
+        display.debug('EXEC: %s' % cmd)
         rc, out, err = run_cmd(cmd, live=True)
 
         if self.options.purge:

lib/ansible/cli/vault.py

@@ -26,6 +26,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.vault import VaultEditor
 from ansible.cli import CLI
+from ansible.utils.unicode import to_unicode
 
 try:
     from __main__ import display
@@ -69,7 +70,7 @@ class VaultCLI(CLI):
         elif self.action == "rekey":
             self.parser.set_usage("usage: %prog rekey [options] file_name")
 
-        self.options, self.args = self.parser.parse_args()
+        self.options, self.args = self.parser.parse_args(self.args[1:])
         display.verbosity = self.options.verbosity
 
         can_output = ['encrypt', 'decrypt']
@@ -157,7 +158,12 @@ class VaultCLI(CLI):
     def execute_view(self):
 
         for f in self.args:
-            self.pager(self.editor.plaintext(f))
+            # Note: vault should return byte strings because it could encrypt
+            # and decrypt binary files.  We are responsible for changing it to
+            # unicode here because we are displaying it and therefore can make
+            # the decision that the display doesn't have to be precisely what
+            # the input was (leave that to decrypt instead)
+            self.pager(to_unicode(self.editor.plaintext(f)))
 
     def execute_rekey(self):
         for f in self.args:

lib/ansible/constants.py

@@ -120,16 +120,20 @@ DEFAULT_COW_WHITELIST = ['bud-frogs', 'bunny', 'cheese', 'daemon', 'default', 'd
 # sections in config file
 DEFAULTS='defaults'
 
+# FIXME: add deprecation warning when these get set
+#### DEPRECATED VARS #### 
+# use more sanely named 'inventory'
 DEPRECATED_HOST_LIST  = get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts', ispath=True)
+# this is not used since 0.5 but people might still have in config
+DEFAULT_PATTERN           = get_config(p, DEFAULTS, 'pattern', None, None)
 
-# generally configurable things
+#### GENERALLY CONFIGURABLE THINGS ####
 DEFAULT_DEBUG             = get_config(p, DEFAULTS, 'debug',            'ANSIBLE_DEBUG',            False, boolean=True)
 DEFAULT_HOST_LIST         = get_config(p, DEFAULTS,'inventory', 'ANSIBLE_INVENTORY', DEPRECATED_HOST_LIST, ispath=True)
 DEFAULT_MODULE_PATH       = get_config(p, DEFAULTS, 'library',          'ANSIBLE_LIBRARY',          None, ispath=True)
 DEFAULT_ROLES_PATH        = get_config(p, DEFAULTS, 'roles_path',       'ANSIBLE_ROLES_PATH',       '/etc/ansible/roles', ispath=True)
 DEFAULT_REMOTE_TMP        = get_config(p, DEFAULTS, 'remote_tmp',       'ANSIBLE_REMOTE_TEMP',      '$HOME/.ansible/tmp')
 DEFAULT_MODULE_NAME       = get_config(p, DEFAULTS, 'module_name',      None,                       'command')
-DEFAULT_PATTERN           = get_config(p, DEFAULTS, 'pattern',          None,                       '*')
 DEFAULT_FORKS             = get_config(p, DEFAULTS, 'forks',            'ANSIBLE_FORKS',            5, integer=True)
 DEFAULT_MODULE_ARGS       = get_config(p, DEFAULTS, 'module_args',      'ANSIBLE_MODULE_ARGS',      '')
 DEFAULT_MODULE_LANG       = get_config(p, DEFAULTS, 'module_lang',      'ANSIBLE_MODULE_LANG',      os.getenv('LANG', 'en_US.UTF-8'))
@@ -152,17 +156,24 @@ DEFAULT_PRIVATE_ROLE_VARS = get_config(p, DEFAULTS, 'private_role_vars', 'ANSIBL
 DEFAULT_JINJA2_EXTENSIONS = get_config(p, DEFAULTS, 'jinja2_extensions', 'ANSIBLE_JINJA2_EXTENSIONS', None)
 DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECUTABLE', '/bin/sh')
 DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
+DEFAULT_GATHER_SUBSET     = get_config(p, DEFAULTS, 'gather_subset', 'ANSIBLE_GATHER_SUBSET', 'all').lower()
 DEFAULT_LOG_PATH          = get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', '', ispath=True)
 DEFAULT_FORCE_HANDLERS    = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
 DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)
 DEFAULT_VAR_COMPRESSION_LEVEL = get_config(p, DEFAULTS, 'var_compression_level', 'ANSIBLE_VAR_COMPRESSION_LEVEL', 0, integer=True)
 
+# static includes
+DEFAULT_TASK_INCLUDES_STATIC    = get_config(p, DEFAULTS, 'task_includes_static', 'ANSIBLE_TASK_INCLUDES_STATIC', False, boolean=True)
+DEFAULT_HANDLER_INCLUDES_STATIC = get_config(p, DEFAULTS, 'handler_includes_static', 'ANSIBLE_HANDLER_INCLUDES_STATIC', False, boolean=True)
+
 # disclosure
 DEFAULT_NO_LOG           = get_config(p, DEFAULTS, 'no_log', 'ANSIBLE_NO_LOG', False, boolean=True)
-DEFAULT_NO_TARGET_SYSLOG   = get_config(p, DEFAULTS, 'no_target_syslog', 'ANSIBLE_NO_TARGET_SYSLOG', True, boolean=True)
+DEFAULT_NO_TARGET_SYSLOG   = get_config(p, DEFAULTS, 'no_target_syslog', 'ANSIBLE_NO_TARGET_SYSLOG', False, boolean=True)
+ALLOW_WORLD_READABLE_TMPFILES = get_config(p, DEFAULTS, 'allow_world_readable_tmpfiles', None, False, boolean=True)
 
 # selinux
 DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True)
+DEFAULT_LIBVIRT_LXC_NOSECLABEL = get_config(p, 'selinux', 'libvirt_lxc_noseclabel', 'LIBVIRT_LXC_NOSECLABEL', False, boolean=True)
 
 ### PRIVILEGE ESCALATION ###
 # Backwards Compat
@@ -197,7 +208,7 @@ DEFAULT_BECOME_ASK_PASS   = get_config(p, 'privilege_escalation', 'become_ask_pa
 # the module takes both, bad things could happen.
 # In the future we should probably generalize this even further
 # (mapping of param: squash field)
-DEFAULT_SQUASH_ACTIONS         = get_config(p, DEFAULTS, 'squash_actions',     'ANSIBLE_SQUASH_ACTIONS', "apt, yum, pkgng, zypper, dnf", islist=True)
+DEFAULT_SQUASH_ACTIONS         = get_config(p, DEFAULTS, 'squash_actions',     'ANSIBLE_SQUASH_ACTIONS', "apk, apt, dnf, package, pacman, pkgng, yum, zypper", islist=True)
 # paths
 DEFAULT_ACTION_PLUGIN_PATH     = get_config(p, DEFAULTS, 'action_plugins',     'ANSIBLE_ACTION_PLUGINS', '~/.ansible/plugins/action:/usr/share/ansible/plugins/action', ispath=True)
 DEFAULT_CACHE_PLUGIN_PATH      = get_config(p, DEFAULTS, 'cache_plugins',      'ANSIBLE_CACHE_PLUGINS', '~/.ansible/plugins/cache:/usr/share/ansible/plugins/cache', ispath=True)
@@ -208,6 +219,7 @@ DEFAULT_INVENTORY_PLUGIN_PATH  = get_config(p, DEFAULTS, 'inventory_plugins',  '
 DEFAULT_VARS_PLUGIN_PATH       = get_config(p, DEFAULTS, 'vars_plugins',       'ANSIBLE_VARS_PLUGINS', '~/.ansible/plugins/vars:/usr/share/ansible/plugins/vars', ispath=True)
 DEFAULT_FILTER_PLUGIN_PATH     = get_config(p, DEFAULTS, 'filter_plugins',     'ANSIBLE_FILTER_PLUGINS', '~/.ansible/plugins/filter:/usr/share/ansible/plugins/filter', ispath=True)
 DEFAULT_TEST_PLUGIN_PATH       = get_config(p, DEFAULTS, 'test_plugins',       'ANSIBLE_TEST_PLUGINS', '~/.ansible/plugins/test:/usr/share/ansible/plugins/test', ispath=True)
+DEFAULT_STRATEGY_PLUGIN_PATH   = get_config(p, DEFAULTS, 'strategy_plugins',   'ANSIBLE_STRATEGY_PLUGINS', '~/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy', ispath=True)
 DEFAULT_STDOUT_CALLBACK        = get_config(p, DEFAULTS, 'stdout_callback',    'ANSIBLE_STDOUT_CALLBACK', 'default')
 # cache
 CACHE_PLUGIN                   = get_config(p, DEFAULTS, 'fact_caching', 'ANSIBLE_CACHE_PLUGIN', 'memory')
@@ -221,6 +233,7 @@ ANSIBLE_NOCOLOR                = get_config(p, DEFAULTS, 'nocolor', 'ANSIBLE_NOC
 ANSIBLE_NOCOWS                 = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None, boolean=True)
 ANSIBLE_COW_SELECTION          = get_config(p, DEFAULTS, 'cow_selection', 'ANSIBLE_COW_SELECTION', 'default')
 ANSIBLE_COW_WHITELIST          = get_config(p, DEFAULTS, 'cow_whitelist', 'ANSIBLE_COW_WHITELIST', DEFAULT_COW_WHITELIST, islist=True)
+DISPLAY_ARGS_TO_STDOUT         = get_config(p, DEFAULTS, 'display_args_to_stdout', 'DISPLAY_ARGS_TO_STDOUT', False, boolean=True)
 DISPLAY_SKIPPED_HOSTS          = get_config(p, DEFAULTS, 'display_skipped_hosts', 'DISPLAY_SKIPPED_HOSTS', True, boolean=True)
 DEFAULT_UNDEFINED_VAR_BEHAVIOR = get_config(p, DEFAULTS, 'error_on_undefined_vars', 'ANSIBLE_ERROR_ON_UNDEFINED_VARS', True, boolean=True)
 HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  'ANSIBLE_HOST_KEY_CHECKING',    True, boolean=True)
@@ -231,8 +244,10 @@ COMMAND_WARNINGS               = get_config(p, DEFAULTS, 'command_warnings', 'AN
 DEFAULT_LOAD_CALLBACK_PLUGINS  = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True)
 DEFAULT_CALLBACK_WHITELIST     = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True)
 RETRY_FILES_ENABLED            = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True)
-RETRY_FILES_SAVE_PATH          = get_config(p, DEFAULTS, 'retry_files_save_path', 'ANSIBLE_RETRY_FILES_SAVE_PATH', '~/', ispath=True)
+RETRY_FILES_SAVE_PATH          = get_config(p, DEFAULTS, 'retry_files_save_path', 'ANSIBLE_RETRY_FILES_SAVE_PATH', None, ispath=True)
 DEFAULT_NULL_REPRESENTATION    = get_config(p, DEFAULTS, 'null_representation', 'ANSIBLE_NULL_REPRESENTATION', None, isnone=True)
+DISPLAY_ARGS_TO_STDOUT         = get_config(p, DEFAULTS, 'display_args_to_stdout', 'ANSIBLE_DISPLAY_ARGS_TO_STDOUT', False, boolean=True)
+MAX_FILE_SIZE_FOR_DIFF         = get_config(p, DEFAULTS, 'max_diff_size', 'ANSIBLE_MAX_DIFF_SIZE', 1024*1024, integer=True)
 
 # CONNECTION RELATED
 ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', '-o ControlMaster=auto -o ControlPersist=60s')
@@ -240,6 +255,7 @@ ANSIBLE_SSH_CONTROL_PATH       = get_config(p, 'ssh_connection', 'control_path',
 ANSIBLE_SSH_PIPELINING         = get_config(p, 'ssh_connection', 'pipelining', 'ANSIBLE_SSH_PIPELINING', False, boolean=True)
 ANSIBLE_SSH_RETRIES            = get_config(p, 'ssh_connection', 'retries', 'ANSIBLE_SSH_RETRIES', 0, integer=True)
 PARAMIKO_RECORD_HOST_KEYS      = get_config(p, 'paramiko_connection', 'record_host_keys', 'ANSIBLE_PARAMIKO_RECORD_HOST_KEYS', True, boolean=True)
+PARAMIKO_PROXY_COMMAND         = get_config(p, 'paramiko_connection', 'proxy_command', 'ANSIBLE_PARAMIKO_PROXY_COMMAND', None)
 
 
 # obsolete -- will be formally removed
@@ -255,12 +271,32 @@ ACCELERATE_MULTI_KEY           = get_config(p, 'accelerate', 'accelerate_multi_k
 PARAMIKO_PTY                   = get_config(p, 'paramiko_connection', 'pty', 'ANSIBLE_PARAMIKO_PTY', True, boolean=True)
 
 # galaxy related
-DEFAULT_GALAXY_URI             = get_config(p, 'galaxy', 'server_uri', 'ANSIBLE_GALAXY_SERVER_URI', 'https://galaxy.ansible.com')
+GALAXY_SERVER                  = get_config(p, 'galaxy', 'server', 'ANSIBLE_GALAXY_SERVER', 'https://galaxy.ansible.com')
+GALAXY_IGNORE_CERTS            = get_config(p, 'galaxy', 'ignore_certs', 'ANSIBLE_GALAXY_IGNORE', False, boolean=True)
 # this can be configured to blacklist SCMS but cannot add new ones unless the code is also updated
 GALAXY_SCMS                    = get_config(p, 'galaxy', 'scms', 'ANSIBLE_GALAXY_SCMS', 'git, hg', islist=True)
 
 # characters included in auto-generated passwords
 DEFAULT_PASSWORD_CHARS = ascii_letters + digits + ".,:-_"
+STRING_TYPE_FILTERS = get_config(p, 'jinja2', 'dont_type_filters', 'ANSIBLE_STRING_TYPE_FILTERS', ['string', 'to_json', 'to_nice_json', 'to_yaml', 'ppretty', 'json'], islist=True )
+
+# colors
+COLOR_HIGHLIGHT   = get_config(p, 'colors', 'highlight', 'ANSIBLE_COLOR_HIGHLIGHT', 'white')
+COLOR_VERBOSE     = get_config(p, 'colors', 'verbose', 'ANSIBLE_COLOR_VERBOSE', 'blue')
+COLOR_WARN        = get_config(p, 'colors', 'warn', 'ANSIBLE_COLOR_WARN', 'bright purple')
+COLOR_ERROR       = get_config(p, 'colors', 'error', 'ANSIBLE_COLOR_ERROR', 'red')
+COLOR_DEBUG       = get_config(p, 'colors', 'debug', 'ANSIBLE_COLOR_DEBUG', 'dark gray')
+COLOR_DEPRECATE   = get_config(p, 'colors', 'deprecate', 'ANSIBLE_COLOR_DEPRECATE', 'purple')
+COLOR_SKIP        = get_config(p, 'colors', 'skip', 'ANSIBLE_COLOR_SKIP', 'cyan')
+COLOR_UNREACHABLE = get_config(p, 'colors', 'unreachable', 'ANSIBLE_COLOR_UNREACHABLE', 'bright red')
+COLOR_OK          = get_config(p, 'colors', 'ok', 'ANSIBLE_COLOR_OK', 'green')
+COLOR_CHANGED     = get_config(p, 'colors', 'ok', 'ANSIBLE_COLOR_CHANGED', 'yellow')
+COLOR_DIFF_ADD    = get_config(p, 'colors', 'diff_add', 'ANSIBLE_COLOR_DIFF_ADD', 'green')
+COLOR_DIFF_REMOVE = get_config(p, 'colors', 'diff_remove', 'ANSIBLE_COLOR_DIFF_REMOVE', 'red')
+COLOR_DIFF_LINES  = get_config(p, 'colors', 'diff_lines', 'ANSIBLE_COLOR_DIFF_LINES', 'cyan')
+
+# diff
+DIFF_CONTEXT = get_config(p, 'diff', 'context', 'ANSIBLE_DIFF_CONTEXT', 3, integer=True)
 
 # non-configurable things
 MODULE_REQUIRE_ARGS       = ['command', 'shell', 'raw', 'script']
@@ -272,6 +308,8 @@ DEFAULT_SUBSET            = None
 DEFAULT_SU_PASS           = None
 VAULT_VERSION_MIN         = 1.0
 VAULT_VERSION_MAX         = 1.0
-MAX_FILE_SIZE_FOR_DIFF    = 1*1024*1024
 TREE_DIR                  = None
 LOCALHOST                 = frozenset(['127.0.0.1', 'localhost', '::1'])
+# module search
+BLACKLIST_EXTS = ('.pyc', '.swp', '.bak', '~', '.rpm', '.md', '.txt')
+IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README", "VERSION", "GUIDELINES", "test-docs.sh"]

lib/ansible/errors/__init__.py

@@ -44,7 +44,7 @@ class AnsibleError(Exception):
     which should be returned by the DataLoader() class.
     '''
 
-    def __init__(self, message="", obj=None, show_content=True):
+    def __init__(self, message="", obj=None, show_content=True, suppress_extended_error=False):
         # we import this here to prevent an import loop problem,
         # since the objects code also imports ansible.errors
         from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
@@ -53,10 +53,12 @@ class AnsibleError(Exception):
         self._show_content = show_content
         if obj and isinstance(obj, AnsibleBaseYAMLObject):
             extended_error = self._get_extended_error()
-            if extended_error:
-                self.message = 'ERROR! %s\n\n%s' % (message, to_str(extended_error))
+            if extended_error and not suppress_extended_error:
+                self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error))
+            else:
+                self.message = '%s' % to_str(message)
         else:
-            self.message = 'ERROR! %s' % message
+            self.message = '%s' % to_str(message)
 
     def __str__(self):
         return self.message

lib/ansible/executor/module_common.py

@@ -21,7 +21,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 # from python and deps
-from ansible.compat.six.moves import StringIO
+from io import BytesIO
 import json
 import os
 import shlex
@@ -30,20 +30,20 @@ import shlex
 from ansible import __version__
 from ansible import constants as C
 from ansible.errors import AnsibleError
-from ansible.utils.unicode import to_bytes
+from ansible.utils.unicode import to_bytes, to_unicode
 
-REPLACER          = "#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
-REPLACER_ARGS     = "\"<<INCLUDE_ANSIBLE_MODULE_ARGS>>\""
-REPLACER_COMPLEX  = "\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
-REPLACER_WINDOWS  = "# POWERSHELL_COMMON"
-REPLACER_WINARGS  = "<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>"
-REPLACER_JSONARGS = "<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
-REPLACER_VERSION  = "\"<<ANSIBLE_VERSION>>\""
-REPLACER_SELINUX  = "<<SELINUX_SPECIAL_FILESYSTEMS>>"
+REPLACER          = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
+REPLACER_ARGS     = b"\"<<INCLUDE_ANSIBLE_MODULE_ARGS>>\""
+REPLACER_COMPLEX  = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
+REPLACER_WINDOWS  = b"# POWERSHELL_COMMON"
+REPLACER_WINARGS  = b"<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>"
+REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
+REPLACER_VERSION  = b"\"<<ANSIBLE_VERSION>>\""
+REPLACER_SELINUX  = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
 
 # We could end up writing out parameters with unicode characters so we need to
 # specify an encoding for the python source file
-ENCODING_STRING = '# -*- coding: utf-8 -*-'
+ENCODING_STRING = b'# -*- coding: utf-8 -*-'
 
 # we've moved the module_common relative to the snippets, so fix the path
 _SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
@@ -53,7 +53,7 @@ _SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
 def _slurp(path):
     if not os.path.exists(path):
         raise AnsibleError("imported module support code does not exist at %s" % path)
-    fd = open(path)
+    fd = open(path, 'rb')
     data = fd.read()
     fd.close()
     return data
@@ -71,49 +71,49 @@ def _find_snippet_imports(module_data, module_path, strip_comments):
         module_style = 'new'
     elif REPLACER_JSONARGS in module_data:
         module_style = 'new'
-    elif 'from ansible.module_utils.' in module_data:
+    elif b'from ansible.module_utils.' in module_data:
         module_style = 'new'
-    elif 'WANT_JSON' in module_data:
+    elif b'WANT_JSON' in module_data:
         module_style = 'non_native_want_json'
 
-    output = StringIO()
-    lines = module_data.split('\n')
+    output = BytesIO()
+    lines = module_data.split(b'\n')
     snippet_names = []
 
     for line in lines:
 
         if REPLACER in line:
             output.write(_slurp(os.path.join(_SNIPPET_PATH, "basic.py")))
-            snippet_names.append('basic')
+            snippet_names.append(b'basic')
         if REPLACER_WINDOWS in line:
             ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
             output.write(ps_data)
-            snippet_names.append('powershell')
-        elif line.startswith('from ansible.module_utils.'):
-            tokens=line.split(".")
+            snippet_names.append(b'powershell')
+        elif line.startswith(b'from ansible.module_utils.'):
+            tokens=line.split(b".")
             import_error = False
             if len(tokens) != 3:
                 import_error = True
-            if " import *" not in line:
+            if b" import *" not in line:
                 import_error = True
             if import_error:
-                raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.basic import *'" % module_path)
+                raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.<lib name> import *'" % module_path)
             snippet_name = tokens[2].split()[0]
             snippet_names.append(snippet_name)
-            output.write(_slurp(os.path.join(_SNIPPET_PATH, snippet_name + ".py")))
+            output.write(_slurp(os.path.join(_SNIPPET_PATH, to_unicode(snippet_name) + ".py")))
         else:
-            if strip_comments and line.startswith("#") or line == '':
+            if strip_comments and line.startswith(b"#") or line == b'':
                 pass
             output.write(line)
-            output.write("\n")
+            output.write(b"\n")
 
     if not module_path.endswith(".ps1"):
         # Unixy modules
-        if len(snippet_names) > 0 and not 'basic' in snippet_names:
+        if len(snippet_names) > 0 and not b'basic' in snippet_names:
             raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path)
     else:
         # Windows modules
-        if len(snippet_names) > 0 and not 'powershell' in snippet_names:
+        if len(snippet_names) > 0 and not b'powershell' in snippet_names:
             raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)
 
     return (output.getvalue(), module_style)
@@ -158,28 +158,28 @@ def modify_module(module_path, module_args, task_vars=dict(), strip_comments=Fal
     # * Cache the modified module?  If only the args are different and we do
     #   that as the last step we could cache all the work up to that point.
 
-    with open(module_path) as f:
+    with open(module_path, 'rb') as f:
 
         # read in the module source
         module_data = f.read()
 
     (module_data, module_style) = _find_snippet_imports(module_data, module_path, strip_comments)
 
-    module_args_json = json.dumps(module_args).encode('utf-8')
-    python_repred_args = repr(module_args_json)
+    module_args_json = to_bytes(json.dumps(module_args))
+    python_repred_args = to_bytes(repr(module_args_json))
 
     # these strings should be part of the 'basic' snippet which is required to be included
-    module_data = module_data.replace(REPLACER_VERSION, repr(__version__))
+    module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
     module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
     module_data = module_data.replace(REPLACER_WINARGS, module_args_json)
     module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
-    module_data = module_data.replace(REPLACER_SELINUX, ','.join(C.DEFAULT_SELINUX_SPECIAL_FS))
+    module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
 
     if module_style == 'new':
         facility = C.DEFAULT_SYSLOG_FACILITY
         if 'ansible_syslog_facility' in task_vars:
             facility = task_vars['ansible_syslog_facility']
-        module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility)
+        module_data = module_data.replace(b'syslog.LOG_USER', to_bytes("syslog.%s" % facility))
 
     lines = module_data.split(b"\n", 1)
     shebang = None
@@ -188,12 +188,13 @@ def modify_module(module_path, module_args, task_vars=dict(), strip_comments=Fal
         args = shlex.split(str(shebang[2:]))
         interpreter = args[0]
         interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
+        interpreter = to_bytes(interpreter)
 
         if interpreter_config in task_vars:
             interpreter = to_bytes(task_vars[interpreter_config], errors='strict')
             lines[0] = shebang = b"#!{0} {1}".format(interpreter, b" ".join(args[1:]))
 
-        if os.path.basename(interpreter).startswith('python'):
+        if os.path.basename(interpreter).startswith(b'python'):
             lines.insert(1, ENCODING_STRING)
     else:
         # No shebang, assume a binary module?

lib/ansible/executor/play_iterator.py

@@ -49,6 +49,7 @@ class HostState:
         self.cur_rescue_task    = 0
         self.cur_always_task    = 0
         self.cur_role           = None
+        self.cur_dep_chain      = None
         self.run_state          = PlayIterator.ITERATING_SETUP
         self.fail_state         = PlayIterator.FAILED_NONE
         self.pending_setup      = False
@@ -57,20 +58,55 @@ class HostState:
         self.always_child_state = None
 
     def __repr__(self):
-        return "HOST STATE: block=%d, task=%d, rescue=%d, always=%d, role=%s, run_state=%d, fail_state=%d, pending_setup=%s, tasks child state? %s, rescue child state? %s, always child state? %s" % (
+        return "HostState(%r)" % self._blocks
+
+    def __str__(self):
+        def _run_state_to_string(n):
+            states = ["ITERATING_SETUP", "ITERATING_TASKS", "ITERATING_RESCUE", "ITERATING_ALWAYS", "ITERATING_COMPLETE"]
+            try:
+                return states[n]
+            except IndexError:
+                return "UNKNOWN STATE"
+
+        def _failed_state_to_string(n):
+            states = {1:"FAILED_SETUP", 2:"FAILED_TASKS", 4:"FAILED_RESCUE", 8:"FAILED_ALWAYS"}
+            if n == 0:
+                return "FAILED_NONE"
+            else:
+                ret = []
+                for i in (1, 2, 4, 8):
+                    if n & i:
+                        ret.append(states[i])
+                return "|".join(ret)
+
+        return "HOST STATE: block=%d, task=%d, rescue=%d, always=%d, role=%s, run_state=%s, fail_state=%s, pending_setup=%s, tasks child state? %s, rescue child state? %s, always child state? %s" % (
             self.cur_block,
             self.cur_regular_task,
             self.cur_rescue_task,
             self.cur_always_task,
             self.cur_role,
-            self.run_state,
-            self.fail_state,
+            _run_state_to_string(self.run_state),
+            _failed_state_to_string(self.fail_state),
             self.pending_setup,
             self.tasks_child_state,
             self.rescue_child_state,
             self.always_child_state,
         )
 
+    def __eq__(self, other):
+        if not isinstance(other, HostState):
+            return False
+
+        for attr in (
+            '_blocks', 'cur_block', 'cur_regular_task', 'cur_rescue_task', 'cur_always_task',
+            'cur_role', 'run_state', 'fail_state', 'pending_setup', 'cur_dep_chain',
+            'tasks_child_state', 'rescue_child_state', 'always_child_state'
+            ):
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+
+        return True
+
     def get_current_block(self):
         return self._blocks[self.cur_block]
 
@@ -84,6 +120,8 @@ class HostState:
         new_state.run_state        = self.run_state
         new_state.fail_state       = self.fail_state
         new_state.pending_setup    = self.pending_setup
+        if self.cur_dep_chain is not None:
+            new_state.cur_dep_chain = self.cur_dep_chain[:]
         if self.tasks_child_state is not None:
             new_state.tasks_child_state = self.tasks_child_state.copy()
         if self.rescue_child_state is not None:
@@ -111,38 +149,63 @@ class PlayIterator:
 
     def __init__(self, inventory, play, play_context, variable_manager, all_vars, start_at_done=False):
         self._play = play
-
         self._blocks = []
+
+        # Default options to gather
+        gather_subset = C.DEFAULT_GATHER_SUBSET
+
+        # Retrieve subset to gather
+        if self._play.gather_subset is not None:
+            gather_subset = self._play.gather_subset
+
+        setup_block = Block(play=self._play)
+        setup_task = Task(block=setup_block)
+        setup_task.action = 'setup'
+        setup_task.tags   = ['always']
+        setup_task.args   = {
+          'gather_subset': gather_subset,
+        }
+        setup_task.set_loader(self._play._loader)
+        setup_block.block = [setup_task]
+
+        setup_block = setup_block.filter_tagged_tasks(play_context, all_vars)
+        self._blocks.append(setup_block)
+
         for block in self._play.compile():
             new_block = block.filter_tagged_tasks(play_context, all_vars)
             if new_block.has_tasks():
                 self._blocks.append(new_block)
 
         self._host_states = {}
+        start_at_matched = False
         for host in inventory.get_hosts(self._play.hosts):
-             self._host_states[host.name] = HostState(blocks=self._blocks)
-             # if the host's name is in the variable manager's fact cache, then set
-             # its _gathered_facts flag to true for smart gathering tests later
-             if host.name in variable_manager._fact_cache:
-                 host._gathered_facts = True
-             # if we're looking to start at a specific task, iterate through
-             # the tasks for this host until we find the specified task
-             if play_context.start_at_task is not None and not start_at_done:
-                 while True:
-                     (s, task) = self.get_next_task_for_host(host, peek=True)
-                     if s.run_state == self.ITERATING_COMPLETE:
-                         break
-                     if task.name == play_context.start_at_task or fnmatch.fnmatch(task.name, play_context.start_at_task) or \
-                        task.get_name() == play_context.start_at_task or fnmatch.fnmatch(task.get_name(), play_context.start_at_task):
-                         # we have our match, so clear the start_at_task field on the
-                         # play context to flag that we've started at a task (and future
-                         # plays won't try to advance)
-                         play_context.start_at_task = None
-                         break
-                     else:
-                         self.get_next_task_for_host(host)
-                 # finally, reset the host's state to ITERATING_SETUP
-                 self._host_states[host.name].run_state = self.ITERATING_SETUP
+            self._host_states[host.name] = HostState(blocks=self._blocks)
+            # if the host's name is in the variable manager's fact cache, then set
+            # its _gathered_facts flag to true for smart gathering tests later
+            if host.name in variable_manager._fact_cache:
+                host._gathered_facts = True
+            # if we're looking to start at a specific task, iterate through
+            # the tasks for this host until we find the specified task
+            if play_context.start_at_task is not None and not start_at_done:
+                while True:
+                    (s, task) = self.get_next_task_for_host(host, peek=True)
+                    if s.run_state == self.ITERATING_COMPLETE:
+                        break
+                    if task.name == play_context.start_at_task or fnmatch.fnmatch(task.name, play_context.start_at_task) or \
+                       task.get_name() == play_context.start_at_task or fnmatch.fnmatch(task.get_name(), play_context.start_at_task):
+                        start_at_matched = True
+                        break
+                    else:
+                        self.get_next_task_for_host(host)
+
+                # finally, reset the host's state to ITERATING_SETUP
+                self._host_states[host.name].run_state = self.ITERATING_SETUP
+
+        if start_at_matched:
+            # we have our match, so clear the start_at_task field on the
+            # play context to flag that we've started at a task (and future
+            # plays won't try to advance)
+            play_context.start_at_task = None
 
         # Extend the play handlers list to include the handlers defined in roles
         self._play.handlers.extend(play.compile_roles_handlers())
@@ -161,41 +224,23 @@ class PlayIterator:
         task = None
         if s.run_state == self.ITERATING_COMPLETE:
             display.debug("host %s is done iterating, returning" % host.name)
-            return (None, None)
-        elif s.run_state == self.ITERATING_SETUP:
-            s.run_state = self.ITERATING_TASKS
-            s.pending_setup = True
+            return (s, None)
 
-            # Gather facts if the default is 'smart' and we have not yet
-            # done it for this host; or if 'explicit' and the play sets
-            # gather_facts to True; or if 'implicit' and the play does
-            # NOT explicitly set gather_facts to False.
+        old_s = s
+        (s, task) = self._get_next_task_from_state(s, host=host, peek=peek)
 
-            gathering = C.DEFAULT_GATHERING
-            implied = self._play.gather_facts is None or boolean(self._play.gather_facts)
-
-            if (gathering == 'implicit' and implied) or \
-               (gathering == 'explicit' and boolean(self._play.gather_facts)) or \
-               (gathering == 'smart' and implied and not host._gathered_facts):
-                if not peek:
-                    # mark the host as having gathered facts
-                    host.set_gathered_facts(True)
-
-                task = Task()
-                task.action = 'setup'
-                task.args   = {}
-                task.set_loader(self._play._loader)
+        def _roles_are_different(ra, rb):
+            if ra != rb:
+                return True
             else:
-                s.pending_setup = False
-
-        if not task:
-            (s, task) = self._get_next_task_from_state(s, peek=peek)
+                return old_s.cur_dep_chain != task._block.get_dep_chain()
 
         if task and task._role:
             # if we had a current role, mark that role as completed
-            if s.cur_role and task._role != s.cur_role and host.name in s.cur_role._had_task_run and not peek:
+            if s.cur_role and _roles_are_different(task._role, s.cur_role) and host.name in s.cur_role._had_task_run and not peek:
                 s.cur_role._completed[host.name] = True
             s.cur_role = task._role
+            s.cur_dep_chain = task._block.get_dep_chain()
 
         if not peek:
             self._host_states[host.name] = s
@@ -206,7 +251,7 @@ class PlayIterator:
         return (s, task)
 
 
-    def _get_next_task_from_state(self, state, peek):
+    def _get_next_task_from_state(self, state, host, peek):
 
         task = None
 
@@ -221,89 +266,155 @@ class PlayIterator:
                 state.run_state = self.ITERATING_COMPLETE
                 return (state, None)
 
-            if state.run_state == self.ITERATING_TASKS:
+            if state.run_state == self.ITERATING_SETUP:
+                # First, we check to see if we were pending setup. If not, this is
+                # the first trip through ITERATING_SETUP, so we set the pending_setup
+                # flag and try to determine if we do in fact want to gather facts for
+                # the specified host.
+                if not state.pending_setup:
+                    state.pending_setup = True
+
+                    # Gather facts if the default is 'smart' and we have not yet
+                    # done it for this host; or if 'explicit' and the play sets
+                    # gather_facts to True; or if 'implicit' and the play does
+                    # NOT explicitly set gather_facts to False.
+
+                    gathering = C.DEFAULT_GATHERING
+                    implied = self._play.gather_facts is None or boolean(self._play.gather_facts)
+
+                    if (gathering == 'implicit' and implied) or \
+                       (gathering == 'explicit' and boolean(self._play.gather_facts)) or \
+                       (gathering == 'smart' and implied and not host._gathered_facts):
+                        # The setup block is always self._blocks[0], as we inject it
+                        # during the play compilation in __init__ above.
+                        setup_block = self._blocks[0]
+                        if setup_block.has_tasks() and len(setup_block.block) > 0:
+                            task = setup_block.block[0]
+                        if not peek:
+                            # mark the host as having gathered facts, because we're
+                            # returning the setup task to be executed
+                            host.set_gathered_facts(True)
+                else:
+                    # This is the second trip through ITERATING_SETUP, so we clear
+                    # the flag and move onto the next block in the list while setting
+                    # the run state to ITERATING_TASKS
+                    state.pending_setup = False
+
+                    state.cur_block += 1
+                    state.cur_regular_task = 0
+                    state.cur_rescue_task  = 0
+                    state.cur_always_task  = 0
+                    state.run_state = self.ITERATING_TASKS
+                    state.child_state = None
+
+            elif state.run_state == self.ITERATING_TASKS:
                 # clear the pending setup flag, since we're past that and it didn't fail
                 if state.pending_setup:
                     state.pending_setup = False
 
-                if state.fail_state & self.FAILED_TASKS == self.FAILED_TASKS:
-                    state.run_state = self.ITERATING_RESCUE
-                elif state.cur_regular_task >= len(block.block):
-                    state.run_state = self.ITERATING_ALWAYS
+                # First, we check for a child task state that is not failed, and if we
+                # have one recurse into it for the next task. If we're done with the child
+                # state, we clear it and drop back to geting the next task from the list.
+                if state.tasks_child_state:
+                    if state.tasks_child_state.fail_state != self.FAILED_NONE:
+                        # failed child state, so clear it and move into the rescue portion
+                        state.tasks_child_state = None
+                        state.fail_state |= self.FAILED_TASKS
+                        state.run_state = self.ITERATING_RESCUE
+                    else:
+                        # get the next task recursively
+                        (state.tasks_child_state, task) = self._get_next_task_from_state(state.tasks_child_state, host=host, peek=peek)
+                        if task is None or state.tasks_child_state.run_state == self.ITERATING_COMPLETE:
+                            # we're done with the child state, so clear it and continue
+                            # back to the top of the loop to get the next task
+                            state.tasks_child_state = None
+                            continue
                 else:
-                    task = block.block[state.cur_regular_task]
-                    # if the current task is actually a child block, we dive into it
-                    if isinstance(task, Block) or state.tasks_child_state is not None:
-                        if state.tasks_child_state is None:
+                    # First here, we check to see if we've failed anywhere down the chain
+                    # of states we have, and if so we move onto the rescue portion. Otherwise,
+                    # we check to see if we've moved past the end of the list of tasks. If so,
+                    # we move into the always portion of the block, otherwise we get the next
+                    # task from the list.
+                    if self._check_failed_state(state):
+                        state.run_state = self.ITERATING_RESCUE
+                    elif state.cur_regular_task >= len(block.block):
+                        state.run_state = self.ITERATING_ALWAYS
+                    else:
+                        task = block.block[state.cur_regular_task]
+                        # if the current task is actually a child block, create a child
+                        # state for us to recurse into on the next pass
+                        if isinstance(task, Block) or state.tasks_child_state is not None:
                             state.tasks_child_state = HostState(blocks=[task])
                             state.tasks_child_state.run_state = self.ITERATING_TASKS
                             state.tasks_child_state.cur_role = state.cur_role
-                        (state.tasks_child_state, task) = self._get_next_task_from_state(state.tasks_child_state, peek=peek)
-                        if task is None:
-                            # check to see if the child state was failed, if so we need to
-                            # fail here too so we don't continue iterating tasks
-                            if state.tasks_child_state.fail_state != self.FAILED_NONE:
-                                state.fail_state |= self.FAILED_TASKS
-                            state.tasks_child_state = None
-                            state.cur_regular_task += 1
-                            continue
-                    else:
+                            # since we've created the child state, clear the task
+                            # so we can pick up the child state on the next pass
+                            task = None
                         state.cur_regular_task += 1
 
             elif state.run_state == self.ITERATING_RESCUE:
-                if state.fail_state & self.FAILED_RESCUE == self.FAILED_RESCUE:
-                    state.run_state = self.ITERATING_ALWAYS
-                elif state.cur_rescue_task >= len(block.rescue):
-                    if len(block.rescue) > 0:
-                        state.fail_state = self.FAILED_NONE
-                    state.run_state = self.ITERATING_ALWAYS
+                # The process here is identical to ITERATING_TASKS, except instead
+                # we move into the always portion of the block.
+                if state.rescue_child_state:
+                    if state.rescue_child_state.fail_state != self.FAILED_NONE:
+                        state.rescue_child_state = None
+                        state.fail_state |= self.FAILED_RESCUE
+                        state.run_state = self.ITERATING_ALWAYS
+                    else:
+                        (state.rescue_child_state, task) = self._get_next_task_from_state(state.rescue_child_state, host=host, peek=peek)
+                        if task is None:
+                            state.rescue_child_state = None
+                            continue
                 else:
-                    task = block.rescue[state.cur_rescue_task]
-                    if isinstance(task, Block) or state.rescue_child_state is not None:
-                        if state.rescue_child_state is None:
+                    if state.fail_state & self.FAILED_RESCUE == self.FAILED_RESCUE:
+                        state.run_state = self.ITERATING_ALWAYS
+                    elif state.cur_rescue_task >= len(block.rescue):
+                        if len(block.rescue) > 0:
+                            state.fail_state = self.FAILED_NONE
+                        state.run_state = self.ITERATING_ALWAYS
+                    else:
+                        task = block.rescue[state.cur_rescue_task]
+                        if isinstance(task, Block) or state.rescue_child_state is not None:
                             state.rescue_child_state = HostState(blocks=[task])
                             state.rescue_child_state.run_state = self.ITERATING_TASKS
                             state.rescue_child_state.cur_role = state.cur_role
-                        (state.rescue_child_state, task) = self._get_next_task_from_state(state.rescue_child_state, peek=peek)
-                        if task is None:
-                            # check to see if the child state was failed, if so we need to
-                            # fail here too so we don't continue iterating rescue
-                            if state.rescue_child_state.fail_state != self.FAILED_NONE:
-                                state.fail_state |= self.FAILED_RESCUE
-                            state.rescue_child_state = None
-                            state.cur_rescue_task += 1
-                            continue
-                    else:
+                            task = None
                         state.cur_rescue_task += 1
 
             elif state.run_state == self.ITERATING_ALWAYS:
-                if state.cur_always_task >= len(block.always):
-                    if state.fail_state != self.FAILED_NONE:
+                # And again, the process here is identical to ITERATING_TASKS, except
+                # instead we either move onto the next block in the list, or we set the
+                # run state to ITERATING_COMPLETE in the event of any errors, or when we
+                # have hit the end of the list of blocks.
+                if state.always_child_state:
+                    if state.always_child_state.fail_state != self.FAILED_NONE:
+                        state.always_child_state = None
+                        state.fail_state |= self.FAILED_ALWAYS
                         state.run_state = self.ITERATING_COMPLETE
                     else:
-                        state.cur_block += 1
-                        state.cur_regular_task = 0
-                        state.cur_rescue_task  = 0
-                        state.cur_always_task  = 0
-                        state.run_state = self.ITERATING_TASKS
-                        state.child_state = None
+                        (state.always_child_state, task) = self._get_next_task_from_state(state.always_child_state, host=host, peek=peek)
+                        if task is None:
+                            state.always_child_state = None
                 else:
-                    task = block.always[state.cur_always_task]
-                    if isinstance(task, Block) or state.always_child_state is not None:
-                        if state.always_child_state is None:
+                    if state.cur_always_task >= len(block.always):
+                        if state.fail_state != self.FAILED_NONE:
+                            state.run_state = self.ITERATING_COMPLETE
+                        else:
+                            state.cur_block += 1
+                            state.cur_regular_task = 0
+                            state.cur_rescue_task  = 0
+                            state.cur_always_task  = 0
+                            state.run_state = self.ITERATING_TASKS
+                            state.tasks_child_state = None
+                            state.rescue_child_state = None
+                            state.always_child_state = None
+                    else:
+                        task = block.always[state.cur_always_task]
+                        if isinstance(task, Block) or state.always_child_state is not None:
                             state.always_child_state = HostState(blocks=[task])
                             state.always_child_state.run_state = self.ITERATING_TASKS
                             state.always_child_state.cur_role = state.cur_role
-                        (state.always_child_state, task) = self._get_next_task_from_state(state.always_child_state, peek=peek)
-                        if task is None:
-                            # check to see if the child state was failed, if so we need to
-                            # fail here too so we don't continue iterating always
-                            if state.always_child_state.fail_state != self.FAILED_NONE:
-                                state.fail_state |= self.FAILED_ALWAYS
-                            state.always_child_state = None
-                            state.cur_always_task += 1
-                            continue
-                    else:
+                            task = None
                         state.cur_always_task += 1
 
             elif state.run_state == self.ITERATING_COMPLETE:
@@ -316,7 +427,7 @@ class PlayIterator:
         return (state, task)
 
     def _set_failed_state(self, state):
-        if state.pending_setup:
+        if state.run_state == self.ITERATING_SETUP:
             state.fail_state |= self.FAILED_SETUP
             state.run_state = self.ITERATING_COMPLETE
         elif state.run_state == self.ITERATING_TASKS:
@@ -324,13 +435,21 @@ class PlayIterator:
                 state.tasks_child_state = self._set_failed_state(state.tasks_child_state)
             else:
                 state.fail_state |= self.FAILED_TASKS
-                state.run_state = self.ITERATING_RESCUE
+                if state._blocks[state.cur_block].rescue:
+                    state.run_state = self.ITERATING_RESCUE
+                elif state._blocks[state.cur_block].always:
+                    state.run_state = self.ITERATING_ALWAYS
+                else:
+                    state.run_state = self.ITERATING_COMPLETE
         elif state.run_state == self.ITERATING_RESCUE:
             if state.rescue_child_state is not None:
                 state.rescue_child_state = self._set_failed_state(state.rescue_child_state)
             else:
                 state.fail_state |= self.FAILED_RESCUE
-                state.run_state = self.ITERATING_ALWAYS
+                if state._blocks[state.cur_block].always:
+                    state.run_state = self.ITERATING_ALWAYS
+                else:
+                    state.run_state = self.ITERATING_COMPLETE
         elif state.run_state == self.ITERATING_ALWAYS:
             if state.always_child_state is not None:
                 state.always_child_state = self._set_failed_state(state.always_child_state)
@@ -347,6 +466,31 @@ class PlayIterator:
     def get_failed_hosts(self):
         return dict((host, True) for (host, state) in iteritems(self._host_states) if state.run_state == self.ITERATING_COMPLETE and state.fail_state != self.FAILED_NONE)
 
+    def _check_failed_state(self, state):
+        if state is None:
+            return False
+        elif state.fail_state != self.FAILED_NONE:
+            if state.run_state == self.ITERATING_RESCUE and state.fail_state&self.FAILED_RESCUE == 0 or \
+               state.run_state == self.ITERATING_ALWAYS and state.fail_state&self.FAILED_ALWAYS == 0:
+                return False
+            else:
+                return True
+        elif state.run_state == self.ITERATING_TASKS and self._check_failed_state(state.tasks_child_state):
+            cur_block = self._blocks[state.cur_block]
+            if len(cur_block.rescue) > 0 and state.fail_state & self.FAILED_RESCUE == 0:
+                return False
+            else:
+                return True
+        elif state.run_state == self.ITERATING_RESCUE and self._check_failed_state(state.rescue_child_state):
+            return True
+        elif state.run_state == self.ITERATING_ALWAYS and self._check_failed_state(state.always_child_state):
+            return True
+        return False
+
+    def is_failed(self, host):
+        s = self.get_host_state(host)
+        return self._check_failed_state(s)
+
     def get_original_task(self, host, task):
         '''
         Finds the task in the task list which matches the UUID of the given task.
@@ -354,7 +498,7 @@ class PlayIterator:
         the different processes, and not all data structures are preserved. This method
         allows us to find the original task passed into the executor engine.
         '''
-        def _search_block(block, task):
+        def _search_block(block):
             '''
             helper method to check a block's task lists (block/rescue/always)
             for a given task uuid. If a Block is encountered in the place of a
@@ -364,32 +508,32 @@ class PlayIterator:
             for b in (block.block, block.rescue, block.always):
                 for t in b:
                     if isinstance(t, Block):
-                        res = _search_block(t, task)
+                        res = _search_block(t)
                         if res:
                             return res
                     elif t._uuid == task._uuid:
                         return t
             return None
 
-        def _search_state(state, task):
+        def _search_state(state):
             for block in state._blocks:
-                res = _search_block(block, task)
+                res = _search_block(block)
                 if res:
                     return res
             for child_state in (state.tasks_child_state, state.rescue_child_state, state.always_child_state):
                 if child_state is not None:
-                    res = _search_state(child_state, task)
+                    res = _search_state(child_state)
                     if res:
                         return res
             return None
 
         s = self.get_host_state(host)
-        res = _search_state(s, task)
+        res = _search_state(s)
         if res:
             return res
 
         for block in self._play.handlers:
-            res = _search_block(block, task)
+            res = _search_block(block)
             if res:
                 return res
 
@@ -397,7 +541,7 @@ class PlayIterator:
 
     def _insert_tasks_into_state(self, state, task_list):
         # if we've failed at all, or if the task list is empty, just return the current state
-        if state.fail_state != self.FAILED_NONE or not task_list:
+        if state.fail_state != self.FAILED_NONE and state.run_state not in (self.ITERATING_RESCUE, self.ITERATING_ALWAYS) or not task_list:
             return state
 
         if state.run_state == self.ITERATING_TASKS:

lib/ansible/executor/playbook_executor.py

@@ -19,19 +19,14 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import getpass
-import locale
 import os
-import signal
-import sys
 
 from ansible.compat.six import string_types
 
+from ansible import constants as C
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.playbook import Playbook
 from ansible.template import Templar
-
-from ansible.utils.encrypt import do_encrypt
 from ansible.utils.unicode import to_unicode
 
 try:
@@ -69,8 +64,6 @@ class PlaybookExecutor:
         may limit the runs to serialized groups, etc.
         '''
 
-        signal.signal(signal.SIGINT, self._cleanup)
-
         result = 0
         entrylist = []
         entry = {}
@@ -89,7 +82,7 @@ class PlaybookExecutor:
 
                 i = 1
                 plays = pb.get_plays()
-                display.vv('%d plays in %s' % (len(plays), playbook_path))
+                display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path)))
 
                 for play in plays:
                     if play._included_path is not None:
@@ -111,13 +104,12 @@ class PlaybookExecutor:
                             salt_size = var.get("salt_size", None)
                             salt      = var.get("salt", None)
 
-                            if vname not in play.vars:
+                            if vname not in self._variable_manager.extra_vars:
                                 if self._tqm:
                                     self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default)
-                                if self._options.syntax:
+                                    play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)
+                                else: # we are either in --list-<option> or syntax check
                                     play.vars[vname] = default
-                                else:
-                                    play.vars[vname] = self._do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)
 
                     # Create a temporary copy of the play here, so we can run post_validate
                     # on it without the templating changes affecting the original object.
@@ -153,9 +145,7 @@ class PlaybookExecutor:
                             # conditions are met, we break out, otherwise we only break out if the entire
                             # batch failed
                             failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts)
-                            if new_play.any_errors_fatal and failed_hosts_count > 0:
-                                break
-                            elif new_play.max_fail_percentage is not None and \
+                            if new_play.max_fail_percentage is not None and \
                                int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
                                 break
                             elif len(batch) == failed_hosts_count:
@@ -177,6 +167,21 @@ class PlaybookExecutor:
 
                 # send the stats callback for this playbook
                 if self._tqm is not None:
+                    if C.RETRY_FILES_ENABLED:
+                        retries = set(self._tqm._failed_hosts.keys())
+                        retries.update(self._tqm._unreachable_hosts.keys())
+                        retries = sorted(retries)
+                        if len(retries) > 0:
+                            if C.RETRY_FILES_SAVE_PATH:
+                                basedir = C.shell_expand(C.RETRY_FILES_SAVE_PATH)
+                            else:
+                                basedir = os.path.dirname(playbook_path)
+
+                            (retry_name, _) = os.path.splitext(os.path.basename(playbook_path))
+                            filename = os.path.join(basedir, "%s.retry" % retry_name)
+                            if self._generate_retry_inventory(filename, retries):
+                                display.display("\tto retry, use: --limit @%s\n" % filename)
+
                     self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
 
                 # if the last result wasn't zero, break out of the playbook file name loop
@@ -188,7 +193,7 @@ class PlaybookExecutor:
 
         finally:
             if self._tqm is not None:
-                self._cleanup()
+                self._tqm.cleanup()
 
         if self._options.syntax:
             display.display("No issues encountered")
@@ -196,9 +201,6 @@ class PlaybookExecutor:
 
         return result
 
-    def _cleanup(self, signum=None, framenum=None):
-        return self._tqm.cleanup()
-
     def _get_serialized_batches(self, play):
         '''
         Returns a list of hosts, subdivided into batches based on
@@ -237,48 +239,19 @@ class PlaybookExecutor:
 
             return serialized_batches
 
-    def _do_var_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
+    def _generate_retry_inventory(self, retry_path, replay_hosts):
+        '''
+        Called when a playbook run fails. It generates an inventory which allows
+        re-running on ONLY the failed hosts.  This may duplicate some variable
+        information in group_vars/host_vars but that is ok, and expected.
+        '''
 
-        if sys.__stdin__.isatty():
-            if prompt and default is not None:
-                msg = "%s [%s]: " % (prompt, default)
-            elif prompt:
-                msg = "%s: " % prompt
-            else:
-                msg = 'input for %s: ' % varname
+        try:
+            with open(retry_path, 'w') as fd:
+                for x in replay_hosts:
+                    fd.write("%s\n" % x)
+        except Exception as e:
+            display.error("Could not create retry file '%s'. The error was: %s" % (retry_path, e))
+            return False
 
-            def do_prompt(prompt, private):
-                if sys.stdout.encoding:
-                    msg = prompt.encode(sys.stdout.encoding)
-                else:
-                    # when piping the output, or at other times when stdout
-                    # may not be the standard file descriptor, the stdout
-                    # encoding may not be set, so default to something sane
-                    msg = prompt.encode(locale.getpreferredencoding())
-                if private:
-                    return getpass.getpass(msg)
-                return raw_input(msg)
-
-            if confirm:
-                while True:
-                    result = do_prompt(msg, private)
-                    second = do_prompt("confirm " + msg, private)
-                    if result == second:
-                        break
-                    display.display("***** VALUES ENTERED DO NOT MATCH ****")
-            else:
-                result = do_prompt(msg, private)
-        else:
-            result = None
-            display.warning("Not prompting as we are not in interactive mode")
-
-        # if result is false and default is not None
-        if not result and default is not None:
-            result = default
-
-        if encrypt:
-            result = do_encrypt(result, encrypt, salt_size, salt)
-
-        # handle utf-8 chars
-        result = to_unicode(result, errors='strict')
-        return result
+        return True

lib/ansible/executor/process/result.py

@@ -65,7 +65,7 @@ class ResultProcess(multiprocessing.Process):
         result = None
         starting_point = self._cur_worker
         while True:
-            (worker_prc, main_q, rslt_q) = self._workers[self._cur_worker]
+            (worker_prc, rslt_q) = self._workers[self._cur_worker]
             self._cur_worker += 1
             if self._cur_worker >= len(self._workers):
                 self._cur_worker = 0
@@ -104,6 +104,21 @@ class ResultProcess(multiprocessing.Process):
                     time.sleep(0.0001)
                     continue
 
+                # send callbacks for 'non final' results
+                if '_ansible_retry' in result._result:
+                    self._send_result(('v2_runner_retry', result))
+                    continue
+                elif '_ansible_item_result' in result._result:
+                    if result.is_failed() or result.is_unreachable():
+                        self._send_result(('v2_runner_item_on_failed', result))
+                    elif result.is_skipped():
+                        self._send_result(('v2_runner_item_on_skipped', result))
+                    else:
+                        self._send_result(('v2_runner_item_on_ok', result))
+                        if 'diff' in result._result:
+                            self._send_result(('v2_on_file_diff', result))
+                    continue
+
                 clean_copy = strip_internal_keys(result._result)
                 if 'invocation' in clean_copy:
                     del clean_copy['invocation']
@@ -163,7 +178,7 @@ class ResultProcess(multiprocessing.Process):
 
             except queue.Empty:
                 pass
-            except (KeyboardInterrupt, IOError, EOFError):
+            except (KeyboardInterrupt, SystemExit, IOError, EOFError):
                 break
             except:
                 # TODO: we should probably send a proper callback here instead of

lib/ansible/executor/process/worker.py

@@ -48,6 +48,7 @@ from ansible.playbook.task import Task
 from ansible.vars.unsafe_proxy import AnsibleJSONUnsafeDecoder
 
 from ansible.utils.debug import debug
+from ansible.utils.unicode import to_unicode
 
 __all__ = ['WorkerProcess']
 
@@ -59,14 +60,18 @@ class WorkerProcess(multiprocessing.Process):
     for reading later.
     '''
 
-    def __init__(self, tqm, main_q, rslt_q, hostvars_manager, loader):
+    def __init__(self, rslt_q, task_vars, host, task, play_context, loader, variable_manager, shared_loader_obj):
 
         super(WorkerProcess, self).__init__()
         # takes a task queue manager as the sole param:
-        self._main_q   = main_q
-        self._rslt_q   = rslt_q
-        self._hostvars = hostvars_manager
-        self._loader   = loader
+        self._rslt_q            = rslt_q
+        self._task_vars         = task_vars
+        self._host              = host
+        self._task              = task
+        self._play_context      = play_context
+        self._loader            = loader
+        self._variable_manager  = variable_manager
+        self._shared_loader_obj = shared_loader_obj
 
         # dupe stdin, if we have one
         self._new_stdin = sys.stdin
@@ -97,73 +102,46 @@ class WorkerProcess(multiprocessing.Process):
         if HAS_ATFORK:
             atfork()
 
-        while True:
-            task = None
-            try:
-                #debug("waiting for work")
-                (host, task, basedir, zip_vars, compressed_vars, play_context, shared_loader_obj) = self._main_q.get(block=False)
+        try:
+            # execute the task and build a TaskResult from the result
+            debug("running TaskExecutor() for %s/%s" % (self._host, self._task))
+            executor_result = TaskExecutor(
+                self._host,
+                self._task,
+                self._task_vars,
+                self._play_context,
+                self._new_stdin,
+                self._loader,
+                self._shared_loader_obj,
+                self._rslt_q
+            ).run()
 
-                if compressed_vars:
-                    job_vars = json.loads(zlib.decompress(zip_vars))
-                else:
-                    job_vars = zip_vars
+            debug("done running TaskExecutor() for %s/%s" % (self._host, self._task))
+            self._host.vars = dict()
+            self._host.groups = []
+            task_result = TaskResult(self._host, self._task, executor_result)
 
-                job_vars['hostvars'] = self._hostvars.hostvars()
+            # put the result on the result queue
+            debug("sending task result")
+            self._rslt_q.put(task_result)
+            debug("done sending task result")
 
-                debug("there's work to be done! got a task/handler to work on: %s" % task)
+        except AnsibleConnectionFailure:
+            self._host.vars = dict()
+            self._host.groups = []
+            task_result = TaskResult(self._host, self._task, dict(unreachable=True))
+            self._rslt_q.put(task_result, block=False)
 
-                # because the task queue manager starts workers (forks) before the
-                # playbook is loaded, set the basedir of the loader inherted by
-                # this fork now so that we can find files correctly
-                self._loader.set_basedir(basedir)
-
-                # Serializing/deserializing tasks does not preserve the loader attribute,
-                # since it is passed to the worker during the forking of the process and
-                # would be wasteful to serialize. So we set it here on the task now, and
-                # the task handles updating parent/child objects as needed.
-                task.set_loader(self._loader)
-
-                # execute the task and build a TaskResult from the result
-                debug("running TaskExecutor() for %s/%s" % (host, task))
-                executor_result = TaskExecutor(
-                    host,
-                    task,
-                    job_vars,
-                    play_context,
-                    self._new_stdin,
-                    self._loader,
-                    shared_loader_obj,
-                ).run()
-                debug("done running TaskExecutor() for %s/%s" % (host, task))
-                task_result = TaskResult(host, task, executor_result)
-
-                # put the result on the result queue
-                debug("sending task result")
-                self._rslt_q.put(task_result)
-                debug("done sending task result")
-
-            except queue.Empty:
-                time.sleep(0.0001)
-            except AnsibleConnectionFailure:
+        except Exception as e:
+            if not isinstance(e, (IOError, EOFError, KeyboardInterrupt, SystemExit)) or isinstance(e, TemplateNotFound):
                 try:
-                    if task:
-                        task_result = TaskResult(host, task, dict(unreachable=True))
-                        self._rslt_q.put(task_result, block=False)
+                    self._host.vars = dict()
+                    self._host.groups = []
+                    task_result = TaskResult(self._host, self._task, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout=''))
+                    self._rslt_q.put(task_result, block=False)
                 except:
-                    break
-            except Exception as e:
-                if isinstance(e, (IOError, EOFError, KeyboardInterrupt)) and not isinstance(e, TemplateNotFound):
-                    break
-                else:
-                    try:
-                        if task:
-                            task_result = TaskResult(host, task, dict(failed=True, exception=traceback.format_exc(), stdout=''))
-                            self._rslt_q.put(task_result, block=False)
-                    except:
-                        debug("WORKER EXCEPTION: %s" % e)
-                        debug("WORKER EXCEPTION: %s" % traceback.format_exc())
-                        break
+                    debug(u"WORKER EXCEPTION: %s" % to_unicode(e))
+                    debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc()))
 
         debug("WORKER PROCESS EXITING")
 
-

lib/ansible/executor/task_executor.py

@@ -24,18 +24,20 @@ import json
 import subprocess
 import sys
 import time
+import traceback
 
 from ansible.compat.six import iteritems, string_types
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure
+from ansible.executor.task_result import TaskResult
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
 from ansible.template import Templar
 from ansible.utils.encrypt import key_for_hostname
 from ansible.utils.listify import listify_lookup_plugin_terms
-from ansible.utils.unicode import to_unicode
-from ansible.vars.unsafe_proxy import UnsafeProxy
+from ansible.utils.unicode import to_unicode, to_bytes
+from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var
 
 try:
     from __main__ import display
@@ -59,7 +61,7 @@ class TaskExecutor:
     # the module
     SQUASH_ACTIONS = frozenset(C.DEFAULT_SQUASH_ACTIONS)
 
-    def __init__(self, host, task, job_vars, play_context, new_stdin, loader, shared_loader_obj):
+    def __init__(self, host, task, job_vars, play_context, new_stdin, loader, shared_loader_obj, rslt_q):
         self._host              = host
         self._task              = task
         self._job_vars          = job_vars
@@ -68,11 +70,14 @@ class TaskExecutor:
         self._loader            = loader
         self._shared_loader_obj = shared_loader_obj
         self._connection        = None
+        self._rslt_q            = rslt_q
 
     def run(self):
         '''
         The main executor entrypoint, where we determine if the specified
-        task requires looping and either runs the task with
+        task requires looping and either runs the task with self._run_loop()
+        or self._execute(). After that, the returned results are parsed and
+        returned as a dict.
         '''
 
         display.debug("in run()")
@@ -140,13 +145,15 @@ class TaskExecutor:
             return res
         except AnsibleError as e:
             return dict(failed=True, msg=to_unicode(e, nonstring='simplerepr'))
+        except Exception as e:
+            return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_unicode(traceback.format_exc()), stdout='')
         finally:
             try:
                 self._connection.close()
             except AttributeError:
                 pass
             except Exception as e:
-                display.debug("error closing connection: %s" % to_unicode(e))
+                display.debug(u"error closing connection: %s" % to_unicode(e))
 
     def _get_loop_items(self):
         '''
@@ -175,21 +182,15 @@ class TaskExecutor:
                     # first_found loops are special.  If the item is undefined
                     # then we want to fall through to the next value rather
                     # than failing.
-                    loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar,
-                            loader=self._loader, fail_on_undefined=False, convert_bare=True)
+                    loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=False, convert_bare=True)
                     loop_terms = [t for t in loop_terms if not templar._contains_vars(t)]
                 else:
                     try:
-                        loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar,
-                                loader=self._loader, fail_on_undefined=True, convert_bare=True)
+                        loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True, convert_bare=True)
                     except AnsibleUndefinedVariable as e:
-                        if 'has no attribute' in str(e):
-                            loop_terms = []
-                            display.deprecated("Skipping task due to undefined attribute, in the future this will be a fatal error.")
-                        else:
-                            raise
-                items = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader,
-                        templar=templar).run(terms=loop_terms, variables=self._job_vars)
+                        loop_terms = []
+                        display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e))
+                items = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar).run(terms=loop_terms, variables=self._job_vars, wantlist=True)
             else:
                 raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop)
 
@@ -231,7 +232,7 @@ class TaskExecutor:
                 tmp_task = self._task.copy()
                 tmp_play_context = self._play_context.copy()
             except AnsibleParserError as e:
-                results.append(dict(failed=True, msg=str(e)))
+                results.append(dict(failed=True, msg=to_unicode(e)))
                 continue
 
             # now we swap the internal task and play context with their copies,
@@ -245,7 +246,9 @@ class TaskExecutor:
             # now update the result with the item info, and append the result
             # to the list of results
             res['item'] = item
-            #TODO: send item results to callback here, instead of all at the end
+            res['_ansible_item_result'] = True
+
+            self._rslt_q.put(TaskResult(self._host, self._task, res), block=False)
             results.append(res)
 
         return results
@@ -268,29 +271,46 @@ class TaskExecutor:
         if len(items) > 0 and task_action in self.SQUASH_ACTIONS:
             if all(isinstance(o, string_types) for o in items):
                 final_items = []
-                name = self._task.args.pop('name', None) or self._task.args.pop('pkg', None)
-                # The user is doing an upgrade or some other operation
-                # that doesn't take name or pkg.
+
+                name = None
+                for allowed in ['name', 'pkg', 'package']:
+                    name = self._task.args.pop(allowed, None)
+                    if name is not None:
+                        break
+
+                # This gets the information to check whether the name field
+                # contains a template that we can squash for
+                template_no_item = template_with_item = None
                 if name:
-                    for item in items:
-                        variables['item'] = item
-                        if self._task.evaluate_conditional(templar, variables):
-                            if templar._contains_vars(name):
+                    if templar._contains_vars(name):
+                        variables['item'] = '\0$'
+                        template_no_item = templar.template(name, variables, cache=False)
+                        variables['item'] = '\0@'
+                        template_with_item = templar.template(name, variables, cache=False)
+                        del variables['item']
+
+                    # Check if the user is doing some operation that doesn't take
+                    # name/pkg or the name/pkg field doesn't have any variables
+                    # and thus the items can't be squashed
+                    if template_no_item != template_with_item:
+                        for item in items:
+                            variables['item'] = item
+                            if self._task.evaluate_conditional(templar, variables):
                                 new_item = templar.template(name, cache=False)
                                 final_items.append(new_item)
-                            else:
-                                final_items.append(item)
-                    self._task.args['name'] = final_items
-                    return [final_items]
+                        self._task.args['name'] = final_items
+                        # Wrap this in a list so that the calling function loop
+                        # executes exactly once
+                        return [final_items]
+                    else:
+                        # Restore the name parameter
+                        self._task.args['name'] = name
             #elif:
                 # Right now we only optimize single entries.  In the future we
                 # could optimize more types:
                 # * lists can be squashed together
                 # * dicts could squash entries that match in all cases except the
                 #   name or pkg field.
-                # Note: we really should be checking that the name or pkg field
-                # contains a template that expands with our with_items values.
-                # If it doesn't then we may break things
         return items
 
     def _execute(self, variables=None):
@@ -316,6 +336,11 @@ class TaskExecutor:
             # do the same kind of post validation step on it here before we use it.
             self._play_context.post_validate(templar=templar)
 
+            # now that the play context is finalized, if the remote_addr is not set
+            # default to using the host's address field as the remote address
+            if not self._play_context.remote_addr:
+                self._play_context.remote_addr = self._host.address
+
             # We also add "magic" variables back into the variables dict to make sure
             # a certain subset of variables exist.
             self._play_context.update_vars(variables)
@@ -362,9 +387,13 @@ class TaskExecutor:
                 self._task.args = variable_params
 
         # get the connection and the handler for this execution
-        if not self._connection or not getattr(self._connection, 'connected', False):
+        if not self._connection or not getattr(self._connection, 'connected', False) or self._play_context.remote_addr != self._connection._play_context.remote_addr:
             self._connection = self._get_connection(variables=variables, templar=templar)
             self._connection.set_host_overrides(host=self._host)
+        else:
+            # if connection is reused, its _play_context is no longer valid and needs
+            # to be replaced with the one templated above, in case other data changed
+            self._connection._play_context = self._play_context
 
         self._handler = self._get_action_handler(connection=self._connection, templar=templar)
 
@@ -387,23 +416,26 @@ class TaskExecutor:
 
         # make a copy of the job vars here, in case we need to update them
         # with the registered variable value later on when testing conditions
-        #vars_copy = variables.copy()
         vars_copy = variables.copy()
 
         display.debug("starting attempt loop")
         result = None
         for attempt in range(retries):
-            if attempt > 0:
-                display.display("FAILED - RETRYING: %s (%d retries left). Result was: %s" % (self._task, retries-attempt, result), color="dark gray")
-                result['attempts'] = attempt + 1
-
             display.debug("running the handler")
             try:
                 result = self._handler.run(task_vars=variables)
             except AnsibleConnectionFailure as e:
-                return dict(unreachable=True, msg=str(e))
+                return dict(unreachable=True, msg=to_unicode(e))
             display.debug("handler run complete")
 
+            # preserve no log
+            result["_ansible_no_log"] = self._play_context.no_log
+
+            # update the local copy of vars with the registered value, if specified,
+            # or any facts which may have been generated by the module execution
+            if self._task.register:
+                vars_copy[self._task.register] = wrap_var(result.copy())
+
             if self._task.async > 0:
                 # the async_wrapper module returns dumped JSON via its stdout
                 # response, so we parse it here and replace the result
@@ -412,31 +444,30 @@ class TaskExecutor:
                         return result
                     result = json.loads(result.get('stdout'))
                 except (TypeError, ValueError) as e:
-                    return dict(failed=True, msg="The async task did not return valid JSON: %s" % str(e))
+                    return dict(failed=True, msg=u"The async task did not return valid JSON: %s" % to_unicode(e))
 
                 if self._task.poll > 0:
                     result = self._poll_async_result(result=result, templar=templar)
 
+                # ensure no log is preserved
+                result["_ansible_no_log"] = self._play_context.no_log
+
             # helper methods for use below in evaluating changed/failed_when
             def _evaluate_changed_when_result(result):
-                if self._task.changed_when is not None:
+                if self._task.changed_when is not None and self._task.changed_when:
                     cond = Conditional(loader=self._loader)
-                    cond.when = [ self._task.changed_when ]
+                    cond.when = self._task.changed_when
                     result['changed'] = cond.evaluate_conditional(templar, vars_copy)
 
             def _evaluate_failed_when_result(result):
-                if self._task.failed_when is not None:
+                if self._task.failed_when:
                     cond = Conditional(loader=self._loader)
-                    cond.when = [ self._task.failed_when ]
+                    cond.when = self._task.failed_when
                     failed_when_result = cond.evaluate_conditional(templar, vars_copy)
                     result['failed_when_result'] = result['failed'] = failed_when_result
-                    return failed_when_result
-                return False
-
-            # update the local copy of vars with the registered value, if specified,
-            # or any facts which may have been generated by the module execution
-            if self._task.register:
-                vars_copy[self._task.register] = result
+                else:
+                    failed_when_result = False
+                return failed_when_result
 
             if 'ansible_facts' in result:
                 vars_copy.update(result['ansible_facts'])
@@ -457,17 +488,24 @@ class TaskExecutor:
                 cond.when = self._task.until
                 if cond.evaluate_conditional(templar, vars_copy):
                     break
-
-                # no conditional check, or it failed, so sleep for the specified time
-                time.sleep(delay)
-
-            elif 'failed' not in result:
-                break
+                else:
+                    # no conditional check, or it failed, so sleep for the specified time
+                    result['attempts'] = attempt + 1
+                    result['retries'] = retries
+                    result['_ansible_retry'] = True
+                    display.debug('Retrying task, attempt %d of %d' % (attempt + 1, retries))
+                    self._rslt_q.put(TaskResult(self._host, self._task, result), block=False)
+                    time.sleep(delay)
+        else:
+            if retries > 1:
+                # we ran out of attempts, so mark the result as failed
+                result['attempts'] = retries
+                result['failed'] = True
 
         # do the final update of the local variables here, for both registered
         # values and any facts which may have been created
         if self._task.register:
-            variables[self._task.register] = result
+            variables[self._task.register] = wrap_var(result)
 
         if 'ansible_facts' in result:
             variables.update(result['ansible_facts'])
@@ -489,9 +527,6 @@ class TaskExecutor:
             for k in ('ansible_host', ):
                 result["_ansible_delegated_vars"][k] = delegated_vars.get(k)
 
-        # preserve no_log setting
-        result["_ansible_no_log"] = self._play_context.no_log
-
         # and return
         display.debug("attempt loop complete, returning result")
         return result
@@ -545,9 +580,6 @@ class TaskExecutor:
         correct connection object from the list of connection plugins
         '''
 
-        if not self._play_context.remote_addr:
-            self._play_context.remote_addr = self._host.address
-
         if self._task.delegate_to is not None:
             # since we're delegating, we don't want to use interpreter values
             # which would have been set for the original target host
@@ -575,7 +607,8 @@ class TaskExecutor:
                 try:
                     cmd = subprocess.Popen(['ssh','-o','ControlPersist'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                     (out, err) = cmd.communicate()
-                    if "Bad configuration option" in err or "Usage:" in err:
+                    err = to_unicode(err)
+                    if u"Bad configuration option" in err or u"Usage:" in err:
                         conn_type = "paramiko"
                 except OSError:
                     conn_type = "paramiko"
@@ -613,7 +646,9 @@ class TaskExecutor:
             try:
                 connection._connect()
             except AnsibleConnectionFailure:
+                display.debug('connection failed, fallback to accelerate')
                 res = handler._execute_module(module_name='accelerate', module_args=accelerate_args, task_vars=variables, delete_remote_tmp=False)
+                display.debug(res)
                 connection._connect()
 
         return connection

lib/ansible/executor/task_queue_manager.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from multiprocessing.managers import SyncManager, DictProxy
 import multiprocessing
 import os
 import tempfile
@@ -27,13 +26,16 @@ import tempfile
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.executor.play_iterator import PlayIterator
-from ansible.executor.process.worker import WorkerProcess
 from ansible.executor.process.result import ResultProcess
 from ansible.executor.stats import AggregateStats
+from ansible.playbook.block import Block
 from ansible.playbook.play_context import PlayContext
 from ansible.plugins import callback_loader, strategy_loader, module_loader
 from ansible.template import Templar
 from ansible.vars.hostvars import HostVars
+from ansible.plugins.callback import CallbackBase
+from ansible.utils.unicode import to_unicode
+from ansible.compat.six import string_types
 
 try:
     from __main__ import display
@@ -56,7 +58,7 @@ class TaskQueueManager:
     which dispatches the Play's tasks to hosts.
     '''
 
-    def __init__(self, inventory, variable_manager, loader, options, passwords, stdout_callback=None):
+    def __init__(self, inventory, variable_manager, loader, options, passwords, stdout_callback=None, run_additional_callbacks=True, run_tree=False):
 
         self._inventory        = inventory
         self._variable_manager = variable_manager
@@ -65,6 +67,8 @@ class TaskQueueManager:
         self._stats            = AggregateStats()
         self.passwords         = passwords
         self._stdout_callback  = stdout_callback
+        self._run_additional_callbacks = run_additional_callbacks
+        self._run_tree         = run_tree
 
         self._callbacks_loaded = False
         self._callback_plugins = []
@@ -96,14 +100,9 @@ class TaskQueueManager:
     def _initialize_processes(self, num):
         self._workers = []
 
-        for i in xrange(num):
-            main_q = multiprocessing.Queue()
+        for i in range(num):
             rslt_q = multiprocessing.Queue()
-
-            prc = WorkerProcess(self, main_q, rslt_q, self._hostvars_manager, self._loader)
-            prc.start()
-
-            self._workers.append((prc, main_q, rslt_q))
+            self._workers.append([None, rslt_q])
 
         self._result_prc = ResultProcess(self._final_q, self._workers)
         self._result_prc.start()
@@ -120,11 +119,18 @@ class TaskQueueManager:
         for key in self._notified_handlers.keys():
             del self._notified_handlers[key]
 
-        # FIXME: there is a block compile helper for this...
+        def _process_block(b):
+            temp_list = []
+            for t in b.block:
+                if isinstance(t, Block):
+                    temp_list.extend(_process_block(t))
+                else:
+                    temp_list.append(t)
+            return temp_list
+
         handler_list = []
         for handler_block in handlers:
-            for handler in handler_block.block:
-                handler_list.append(handler)
+            handler_list.extend(_process_block(handler_block))
 
         # then initialize it with the handler names from the handler list
         for handler in handler_list:
@@ -144,8 +150,16 @@ class TaskQueueManager:
         if self._stdout_callback is None:
             self._stdout_callback = C.DEFAULT_STDOUT_CALLBACK
 
-        if self._stdout_callback not in callback_loader:
-            raise AnsibleError("Invalid callback for stdout specified: %s" % self._stdout_callback)
+        if isinstance(self._stdout_callback, CallbackBase):
+            stdout_callback_loaded = True
+        elif isinstance(self._stdout_callback, string_types):
+            if self._stdout_callback not in callback_loader:
+                raise AnsibleError("Invalid callback for stdout specified: %s" % self._stdout_callback)
+            else:
+                self._stdout_callback = callback_loader.get(self._stdout_callback)
+                stdout_callback_loaded = True
+        else:
+            raise AnsibleError("callback must be an instance of CallbackBase or the name of a callback plugin")
 
         for callback_plugin in callback_loader.all(class_only=True):
             if hasattr(callback_plugin, 'CALLBACK_VERSION') and callback_plugin.CALLBACK_VERSION >= 2.0:
@@ -159,7 +173,9 @@ class TaskQueueManager:
                     if callback_name != self._stdout_callback or stdout_callback_loaded:
                         continue
                     stdout_callback_loaded = True
-                elif callback_needs_whitelist and (C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST):
+                elif callback_name == 'tree' and self._run_tree:
+                    pass
+                elif not self._run_additional_callbacks or (callback_needs_whitelist and (C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)):
                     continue
 
             self._callback_plugins.append(callback_plugin())
@@ -184,31 +200,12 @@ class TaskQueueManager:
         new_play = play.copy()
         new_play.post_validate(templar)
 
-        class HostVarsManager(SyncManager):
-            pass
-
-        hostvars = HostVars(
+        self.hostvars = HostVars(
             inventory=self._inventory,
             variable_manager=self._variable_manager,
             loader=self._loader,
         )
 
-        HostVarsManager.register(
-            'hostvars',
-            callable=lambda: hostvars,
-            # FIXME: this is the list of exposed methods to the DictProxy object, plus our
-            #        special ones (set_variable_manager/set_inventory). There's probably a better way
-            #        to do this with a proper BaseProxy/DictProxy derivative
-            exposed=(
-                'set_variable_manager', 'set_inventory', '__contains__', '__delitem__',
-                'set_nonpersistent_facts', 'set_host_facts', 'set_host_variable',
-                '__getitem__', '__len__', '__setitem__', 'clear', 'copy', 'get', 'has_key',
-                'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
-            ),
-        )
-        self._hostvars_manager = HostVarsManager()
-        self._hostvars_manager.start()
-
         # Fork # of forks, # of hosts or serial, whichever is lowest
         contenders =  [self._options.forks, play.serial, len(self._inventory.get_hosts(new_play.hosts))]
         contenders =  [ v for v in contenders if v is not None and v > 0 ]
@@ -248,7 +245,6 @@ class TaskQueueManager:
         # and run the play using the strategy and cleanup on way out
         play_return = strategy.run(iterator, play_context)
         self._cleanup_processes()
-        self._hostvars_manager.shutdown()
         return play_return
 
     def cleanup(self):
@@ -261,10 +257,13 @@ class TaskQueueManager:
         if self._result_prc:
             self._result_prc.terminate()
 
-            for (worker_prc, main_q, rslt_q) in self._workers:
+            for (worker_prc, rslt_q) in self._workers:
                 rslt_q.close()
-                main_q.close()
-                worker_prc.terminate()
+                if worker_prc and worker_prc.is_alive():
+                    try:
+                        worker_prc.terminate()
+                    except AttributeError:
+                        pass
 
     def clear_failed_hosts(self):
         self._failed_hosts = dict()
@@ -288,22 +287,36 @@ class TaskQueueManager:
         self._terminated = True
 
     def send_callback(self, method_name, *args, **kwargs):
-        for callback_plugin in self._callback_plugins:
+        for callback_plugin in [self._stdout_callback] + self._callback_plugins:
             # a plugin that set self.disabled to True will not be called
             # see osx_say.py example for such a plugin
             if getattr(callback_plugin, 'disabled', False):
                 continue
-            methods = [
-                getattr(callback_plugin, method_name, None),
-                getattr(callback_plugin, 'v2_on_any', None)
-            ]
+
+            # try to find v2 method, fallback to v1 method, ignore callback if no method found
+            methods = []
+            for possible in [method_name, 'v2_on_any']:
+                gotit =  getattr(callback_plugin, possible, None)
+                if gotit is None:
+                    gotit = getattr(callback_plugin, possible.replace('v2_',''), None)
+                if gotit is not None:
+                    methods.append(gotit)
+
             for method in methods:
-                if method is not None:
-                    try:
+                try:
+                    # temporary hack, required due to a change in the callback API, so
+                    # we don't break backwards compatibility with callbacks which were
+                    # designed to use the original API
+                    # FIXME: target for removal and revert to the original code here after a year (2017-01-14)
+                    if method_name == 'v2_playbook_on_start':
+                        import inspect
+                        (f_args, f_varargs, f_keywords, f_defaults) = inspect.getargspec(method)
+                        if 'playbook' in f_args:
+                            method(*args, **kwargs)
+                        else:
+                            method()
+                    else:
                         method(*args, **kwargs)
-                    except Exception as e:
-                        try:
-                            v1_method = method.replace('v2_','')
-                            v1_method(*args, **kwargs)
-                        except Exception:
-                            display.warning('Error when using %s: %s' % (method, str(e)))
+                except Exception as e:
+                    #TODO: add config toggle to make this fatal or not?
+                    display.warning(u"Failure when attempting to use callback plugin (%s): %s" % (to_unicode(callback_plugin), to_unicode(e)))

lib/ansible/galaxy/__init__.py

@@ -49,9 +49,34 @@ class Galaxy(object):
         this_dir, this_filename = os.path.split(__file__)
         self.DATA_PATH = os.path.join(this_dir, "data")
 
-        #TODO: move to getter for lazy loading
-        self.default_readme = self._str_from_data_file('readme')
-        self.default_meta = self._str_from_data_file('metadata_template.j2')
+        self._default_readme = None
+        self._default_meta = None
+        self._default_test = None
+        self._default_travis = None
+
+    @property
+    def default_readme(self):
+        if self._default_readme is None:
+            self._default_readme = self._str_from_data_file('readme')
+        return self._default_readme
+
+    @property
+    def default_meta(self):
+        if self._default_meta is None:
+            self._default_meta = self._str_from_data_file('metadata_template.j2')
+        return self._default_meta
+
+    @property
+    def default_test(self):
+        if self._default_test is None:
+            self._default_test = self._str_from_data_file('test_playbook.j2')
+        return self._default_test
+
+    @property
+    def default_travis(self):
+        if self._default_travis is None:
+            self._default_travis = self._str_from_data_file('travis.j2')
+        return self._default_travis
 
     def add_role(self, role):
         self.roles[role.name] = role

lib/ansible/galaxy/api.py

@@ -25,11 +25,15 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import json
+import urllib
+
 from urllib2 import quote as urlquote, HTTPError
 from urlparse import urlparse
 
+import ansible.constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.urls import open_url
+from ansible.galaxy.token import GalaxyToken
 
 try:
     from __main__ import display
@@ -43,45 +47,111 @@ class GalaxyAPI(object):
 
     SUPPORTED_VERSIONS = ['v1']
 
-    def __init__(self, galaxy, api_server):
-
+    def __init__(self, galaxy):
         self.galaxy = galaxy
+        self.token = GalaxyToken()
+        self._api_server = C.GALAXY_SERVER
+        self._validate_certs = not C.GALAXY_IGNORE_CERTS
 
-        try:
-            urlparse(api_server, scheme='https')
-        except:
-            raise AnsibleError("Invalid server API url passed: %s" % api_server)
+        # set validate_certs
+        if galaxy.options.ignore_certs:
+            self._validate_certs = False
+        display.vvv('Validate TLS certificates: %s' % self._validate_certs)
 
-        server_version = self.get_server_api_version('%s/api/' % (api_server))
-        if not server_version:
-            raise AnsibleError("Could not retrieve server API version: %s" % api_server)
+        # set the API server
+        if galaxy.options.api_server != C.GALAXY_SERVER:
+            self._api_server = galaxy.options.api_server
+        display.vvv("Connecting to galaxy_server: %s" % self._api_server)
 
-        if server_version in self.SUPPORTED_VERSIONS:
-            self.baseurl = '%s/api/%s' % (api_server, server_version)
-            self.version = server_version # for future use
-            display.vvvvv("Base API: %s" % self.baseurl)
-        else:
+        server_version = self.get_server_api_version()
+        if not server_version in self.SUPPORTED_VERSIONS:
             raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version)
 
-    def get_server_api_version(self, api_server):
+        self.baseurl = '%s/api/%s' % (self._api_server, server_version)
+        self.version = server_version # for future use
+        display.vvv("Base API: %s" % self.baseurl)           
+
+    def __auth_header(self):
+        token = self.token.get()
+        if token is None:
+            raise AnsibleError("No access token. You must first use login to authenticate and obtain an access token.")
+        return {'Authorization': 'Token ' + token}
+
+    def __call_galaxy(self, url, args=None, headers=None, method=None):
+        if args and not headers:
+            headers = self.__auth_header()
+        try:
+            display.vvv(url)
+            resp = open_url(url, data=args, validate_certs=self._validate_certs, headers=headers, method=method)
+            data = json.load(resp)
+        except HTTPError as e:
+            res = json.load(e)
+            raise AnsibleError(res['detail'])
+        return data
+
+    @property
+    def api_server(self):
+        return self._api_server    
+    
+    @property
+    def validate_certs(self):
+        return self._validate_certs
+
+    def get_server_api_version(self):
         """
         Fetches the Galaxy API current version to ensure
         the API server is up and reachable.
         """
-        #TODO: fix galaxy server which returns current_version path (/api/v1) vs actual version (v1)
-        # also should set baseurl using supported_versions which has path
-        return 'v1'
-
         try:
-            data = json.load(open_url(api_server, validate_certs=self.galaxy.options.validate_certs))
-            return data.get("current_version", 'v1')
-        except Exception:
-            # TODO: report error
-            return None
+            url = '%s/api/' % self._api_server
+            data = json.load(open_url(url, validate_certs=self._validate_certs))
+            return data['current_version']
+        except Exception as e:
+            raise AnsibleError("The API server (%s) is not responding, please try again later." % url)
+        
+    def authenticate(self, github_token):
+        """
+        Retrieve an authentication token
+        """
+        url = '%s/tokens/' % self.baseurl
+        args = urllib.urlencode({"github_token": github_token})
+        resp = open_url(url, data=args, validate_certs=self._validate_certs, method="POST")
+        data = json.load(resp)
+        return data
 
+    def create_import_task(self, github_user, github_repo, reference=None):
+        """
+        Post an import request
+        """
+        url = '%s/imports/' % self.baseurl
+        args = urllib.urlencode({
+            "github_user": github_user,
+            "github_repo": github_repo,
+            "github_reference": reference if reference else ""
+        })
+        data = self.__call_galaxy(url, args=args)
+        if data.get('results', None):
+            return data['results']
+        return data
+
+    def get_import_task(self, task_id=None, github_user=None, github_repo=None):
+        """
+        Check the status of an import task.
+        """
+        url = '%s/imports/' % self.baseurl
+        if not task_id is None:
+            url = "%s?id=%d" % (url,task_id)
+        elif not github_user is None and not github_repo is None:
+            url = "%s?github_user=%s&github_repo=%s" % (url,github_user,github_repo)
+        else:
+            raise AnsibleError("Expected task_id or github_user and github_repo")
+        
+        data = self.__call_galaxy(url)
+        return data['results']
+       
     def lookup_role_by_name(self, role_name, notify=True):
         """
-        Find a role by name
+        Find a role by name.
         """
         role_name = urlquote(role_name)
 
@@ -92,18 +162,12 @@ class GalaxyAPI(object):
             if notify:
                 display.display("- downloading role '%s', owned by %s" % (role_name, user_name))
         except:
-            raise AnsibleError("- invalid role name (%s). Specify role as format: username.rolename" % role_name)
+            raise AnsibleError("Invalid role name (%s). Specify role as format: username.rolename" % role_name)
 
         url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl, user_name, role_name)
-        display.vvvv("- %s" % (url))
-        try:
-            data = json.load(open_url(url, validate_certs=self.galaxy.options.validate_certs))
-            if len(data["results"]) != 0:
-                return data["results"][0]
-        except:
-            # TODO: report on connection/availability errors
-            pass
-
+        data = self.__call_galaxy(url)
+        if len(data["results"]) != 0:
+            return data["results"][0]
         return None
 
     def fetch_role_related(self, related, role_id):
@@ -114,15 +178,14 @@ class GalaxyAPI(object):
 
         try:
             url = '%s/roles/%d/%s/?page_size=50' % (self.baseurl, int(role_id), related)
-            data = json.load(open_url(url, validate_certs=self.galaxy.options.validate_certs))
+            data = self.__call_galaxy(url)
             results = data['results']
-            done = (data.get('next', None) is None)
+            done = (data.get('next_link', None) is None)
             while not done:
-                url = '%s%s' % (self.baseurl, data['next'])
-                display.display(url)
-                data = json.load(open_url(url, validate_certs=self.galaxy.options.validate_certs))
+                url = '%s%s' % (self._api_server, data['next_link'])
+                data = self.__call_galaxy(url)
                 results += data['results']
-                done = (data.get('next', None) is None)
+                done = (data.get('next_link', None) is None)
             return results
         except:
             return None
@@ -131,54 +194,76 @@ class GalaxyAPI(object):
         """
         Fetch the list of items specified.
         """
-
         try:
             url = '%s/%s/?page_size' % (self.baseurl, what)
-            data = json.load(open_url(url, validate_certs=self.galaxy.options.validate_certs))
+            data = self.__call_galaxy(url)
             if "results" in data:
                 results = data['results']
             else:
                 results = data
             done = True
             if "next" in data:
-                done = (data.get('next', None) is None)
+                done = (data.get('next_link', None) is None)
             while not done:
-                url = '%s%s' % (self.baseurl, data['next'])
-                display.display(url)
-                data = json.load(open_url(url, validate_certs=self.galaxy.options.validate_certs))
+                url = '%s%s' % (self._api_server, data['next_link'])
+                data = self.__call_galaxy(url)
                 results += data['results']
-                done = (data.get('next', None) is None)
+                done = (data.get('next_link', None) is None)
             return results
         except Exception as error:
             raise AnsibleError("Failed to download the %s list: %s" % (what, str(error)))
 
-    def search_roles(self, search, platforms=None, tags=None):
+    def search_roles(self, search, **kwargs):
 
-        search_url = self.baseurl + '/roles/?page=1'
+        search_url = self.baseurl + '/search/roles/?'
 
         if search:
-            search_url += '&search=' + urlquote(search)
+            search_url += '&autocomplete=' + urlquote(search)
 
-        if tags is None:
-            tags = []
-        elif isinstance(tags, basestring):
+        tags = kwargs.get('tags',None)
+        platforms = kwargs.get('platforms', None)
+        page_size = kwargs.get('page_size', None)
+        author = kwargs.get('author', None)
+
+        if tags and isinstance(tags, basestring):
             tags = tags.split(',')
-
-        for tag in tags:
-            search_url += '&chain__tags__name=' + urlquote(tag)
-
-        if platforms is None:
-            platforms = []
-        elif isinstance(platforms, basestring):
+            search_url += '&tags_autocomplete=' + '+'.join(tags)
+        
+        if platforms and isinstance(platforms, basestring):
             platforms = platforms.split(',')
+            search_url += '&platforms_autocomplete=' + '+'.join(platforms)
 
-        for plat in platforms:
-            search_url += '&chain__platforms__name=' + urlquote(plat)
-
-        display.debug("Executing query: %s" % search_url)
-        try:
-            data = json.load(open_url(search_url, validate_certs=self.galaxy.options.validate_certs))
-        except HTTPError as e:
-            raise AnsibleError("Unsuccessful request to server: %s" % str(e))
+        if page_size:
+            search_url += '&page_size=%s' % page_size
 
+        if author:
+            search_url += '&username_autocomplete=%s' % author
+   
+        data = self.__call_galaxy(search_url)
+        return data
+
+    def add_secret(self, source, github_user, github_repo, secret):
+        url = "%s/notification_secrets/" % self.baseurl
+        args = urllib.urlencode({
+            "source": source,
+            "github_user": github_user,
+            "github_repo": github_repo,
+            "secret": secret
+        })
+        data = self.__call_galaxy(url, args=args)
+        return data
+
+    def list_secrets(self):
+        url = "%s/notification_secrets" % self.baseurl
+        data = self.__call_galaxy(url, headers=self.__auth_header())
+        return data
+
+    def remove_secret(self, secret_id):
+        url = "%s/notification_secrets/%s/" % (self.baseurl, secret_id)
+        data = self.__call_galaxy(url, headers=self.__auth_header(), method='DELETE')
+        return data
+
+    def delete_role(self, github_user, github_repo):
+        url = "%s/removerole/?github_user=%s&github_repo=%s" % (self.baseurl,github_user,github_repo)
+        data = self.__call_galaxy(url, headers=self.__auth_header(), method='DELETE')
         return data

lib/ansible/galaxy/data/metadata_template.j2

@@ -1,10 +1,12 @@
 galaxy_info:
   author: {{ author }}
-  description: {{description}}
+  description: {{ description }}
   company: {{ company }}
+
   # If the issue tracker for your role is not on github, uncomment the
   # next line and provide a value
   # issue_tracker_url: {{ issue_tracker_url }}
+
   # Some suggested licenses:
   # - BSD (default)
   # - MIT
@@ -13,7 +15,17 @@ galaxy_info:
   # - Apache
   # - CC-BY
   license: {{ license }}
+
   min_ansible_version: {{ min_ansible_version }}
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If travis integration is cofigured, only notification for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+  #github_branch:
+
   #
   # Below are all platforms currently available. Just uncomment
   # the ones that apply to your role. If you don't see your
@@ -26,8 +38,9 @@ galaxy_info:
   #  - all
     {%- for version in versions %}
   #  - {{ version }}
-    {%- endfor %}
+    {%- endfor -%}
   {%- endfor %}
+
   galaxy_tags: []
     # List tags for your role here, one per line. A tag is
     # a keyword that describes and categorizes the role.
@@ -36,6 +49,7 @@ galaxy_info:
     #
     # NOTE: A tag is limited to a single word comprised of
     # alphanumeric characters. Maximum 20 tags per role.
+
 dependencies: []
   # List your role dependencies here, one per line.
   # Be sure to remove the '[]' above if you add dependencies

lib/ansible/galaxy/data/test_playbook.j2

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - {{ role_name }}

lib/ansible/galaxy/data/travis.j2

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

lib/ansible/galaxy/login.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+
+########################################################################
+#
+# (C) 2015, Chris Houseknecht <chouse@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+########################################################################
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import getpass
+import json
+import urllib
+
+from urllib2 import quote as urlquote, HTTPError
+from urlparse import urlparse
+
+from ansible.errors import AnsibleError, AnsibleOptionsError
+from ansible.module_utils.urls import open_url
+from ansible.utils.color import stringc
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+class GalaxyLogin(object):
+    ''' Class to handle authenticating user with Galaxy API prior to performing CUD operations '''
+
+    GITHUB_AUTH = 'https://api.github.com/authorizations'
+    
+    def __init__(self, galaxy, github_token=None):
+        self.galaxy = galaxy
+        self.github_username = None
+        self.github_password = None
+        
+        if github_token == None:
+            self.get_credentials()
+
+    def get_credentials(self):
+        display.display(u'\n\n' + "We need your " + stringc("Github login",'bright cyan') + 
+            " to identify you.", screen_only=True)
+        display.display("This information will " + stringc("not be sent to Galaxy",'bright cyan') + 
+            ", only to " + stringc("api.github.com.","yellow"), screen_only=True)
+        display.display("The password will not be displayed." + u'\n\n', screen_only=True)
+        display.display("Use " + stringc("--github-token",'yellow') + 
+            " if you do not want to enter your password." + u'\n\n', screen_only=True)
+        
+        try:
+            self.github_username = raw_input("Github Username: ")
+        except:
+            pass
+
+        try:
+            self.github_password = getpass.getpass("Password for %s: " % self.github_username)
+        except:
+            pass
+
+        if not self.github_username or not self.github_password:
+            raise AnsibleError("Invalid Github credentials. Username and password are required.")
+
+    def remove_github_token(self):
+        '''
+        If for some reason an ansible-galaxy token was left from a prior login, remove it. We cannot
+        retrieve the token after creation, so we are forced to create a new one.
+        '''
+        try:
+            tokens = json.load(open_url(self.GITHUB_AUTH, url_username=self.github_username,
+                url_password=self.github_password, force_basic_auth=True,))
+        except HTTPError as e:
+            res = json.load(e)
+            raise AnsibleError(res['message'])
+
+        for token in tokens:
+            if token['note'] == 'ansible-galaxy login':
+                display.vvvvv('removing token: %s' % token['token_last_eight'])
+                try: 
+                    open_url('https://api.github.com/authorizations/%d' % token['id'], url_username=self.github_username,
+                        url_password=self.github_password, method='DELETE', force_basic_auth=True,)
+                except HTTPError as e:
+                    res = json.load(e)
+                    raise AnsibleError(res['message'])
+
+    def create_github_token(self):
+        '''
+        Create a personal authorization token with a note of 'ansible-galaxy login'
+        '''
+        self.remove_github_token()
+        args = json.dumps({"scopes":["public_repo"], "note":"ansible-galaxy login"})
+        try:
+            data = json.load(open_url(self.GITHUB_AUTH, url_username=self.github_username,
+                url_password=self.github_password, force_basic_auth=True, data=args))
+        except HTTPError as e:
+            res = json.load(e)
+            raise AnsibleError(res['message'])
+        return data['token']

lib/ansible/galaxy/role.py

@@ -46,7 +46,7 @@ class GalaxyRole(object):
     SUPPORTED_SCMS = set(['git', 'hg'])
     META_MAIN = os.path.join('meta', 'main.yml')
     META_INSTALL = os.path.join('meta', '.galaxy_install_info')
-    ROLE_DIRS = ('defaults','files','handlers','meta','tasks','templates','vars')
+    ROLE_DIRS = ('defaults','files','handlers','meta','tasks','templates','vars','tests')
 
 
     def __init__(self, galaxy, name, src=None, version=None, scm=None, path=None):
@@ -130,13 +130,11 @@ class GalaxyRole(object):
             install_date=datetime.datetime.utcnow().strftime("%c"),
         )
         info_path = os.path.join(self.path, self.META_INSTALL)
-        try:
-            f = open(info_path, 'w+')
-            self._install_info = yaml.safe_dump(info, f)
-        except:
-            return False
-        finally:
-            f.close()
+        with open(info_path, 'w+') as f:
+            try:
+                self._install_info = yaml.safe_dump(info, f)
+            except:
+                return False
 
         return True
 
@@ -198,10 +196,10 @@ class GalaxyRole(object):
                 role_data = self.src
                 tmp_file = self.fetch(role_data)
             else:
-                api = GalaxyAPI(self.galaxy, self.options.api_server)
+                api = GalaxyAPI(self.galaxy)
                 role_data = api.lookup_role_by_name(self.src)
                 if not role_data:
-                    raise AnsibleError("- sorry, %s was not found on %s." % (self.src, self.options.api_server))
+                    raise AnsibleError("- sorry, %s was not found on %s." % (self.src, api.api_server))
 
                 role_versions = api.fetch_role_related('versions', role_data['id'])
                 if not self.version:
@@ -213,8 +211,10 @@ class GalaxyRole(object):
                         loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
                         loose_versions.sort()
                         self.version = str(loose_versions[-1])
+                    elif role_data.get('github_branch', None):
+                        self.version = role_data['github_branch']
                     else:
-                        self.version = 'master'
+                        self.version = 'master' 
                 elif self.version != 'master':
                     if role_versions and self.version not in [a.get('name', None) for a in role_versions]:
                         raise AnsibleError("- the specified version (%s) of %s was not found in the list of available versions (%s)." % (self.version, self.name, role_versions))
@@ -310,5 +310,3 @@ class GalaxyRole(object):
         }
         """
         return dict(scm=self.scm, src=self.src, version=self.version, name=self.name)
-
-

lib/ansible/galaxy/token.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+########################################################################
+#
+# (C) 2015, Chris Houseknecht <chouse@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+########################################################################
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import yaml
+from stat import *
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class GalaxyToken(object):
+    ''' Class to storing and retrieving token in ~/.ansible_galaxy '''
+
+    def __init__(self):
+        self.file = os.path.expanduser("~") + '/.ansible_galaxy'
+        self.config = yaml.safe_load(self.__open_config_for_read())
+        if not self.config:
+            self.config = {}
+        
+    def __open_config_for_read(self):
+        if os.path.isfile(self.file):
+            display.vvv('Opened %s' % self.file)
+            return open(self.file, 'r')
+        # config.yml not found, create and chomd u+rw
+        f = open(self.file,'w')
+        f.close()
+        os.chmod(self.file,S_IRUSR|S_IWUSR) # owner has +rw
+        display.vvv('Created %s' % self.file) 
+        return open(self.file, 'r')
+
+    def set(self, token): 
+        self.config['token'] = token
+        self.save()
+    
+    def get(self):
+        return self.config.get('token', None)
+    
+    def save(self):
+        with open(self.file,'w') as f:
+            yaml.safe_dump(self.config,f,default_flow_style=False)
+        

lib/ansible/inventory/__init__.py

@@ -78,6 +78,10 @@ class Inventory(object):
         self._restriction = None
         self._subset = None
 
+        # clear the cache here, which is only useful if more than
+        # one Inventory objects are created when using the API directly
+        self.clear_pattern_cache()
+
         self.parse_inventory(host_list)
 
     def serialize(self):
@@ -109,7 +113,12 @@ class Inventory(object):
             pass
         elif isinstance(host_list, list):
             for h in host_list:
-                (host, port) = parse_address(h, allow_ranges=False)
+                try:
+                    (host, port) = parse_address(h, allow_ranges=False)
+                except AnsibleError as e:
+                    display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_unicode(e))
+                    host = h
+                    port = None
                 all.add_host(Host(host, port))
         elif self._loader.path_exists(host_list):
             #TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
@@ -124,6 +133,8 @@ class Inventory(object):
             if not self.parser:
                 # should never happen, but JIC
                 raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
+        else:
+            display.warning("Host file not found: %s" % to_unicode(host_list))
 
         self._vars_plugins = [ x for x in vars_loader.all(self) ]
 
@@ -178,25 +189,26 @@ class Inventory(object):
             if self._restriction:
                 pattern_hash += u":%s" % to_unicode(self._restriction)
 
-        if pattern_hash in HOSTS_PATTERNS_CACHE:
-            return HOSTS_PATTERNS_CACHE[pattern_hash][:]
+        if pattern_hash not in HOSTS_PATTERNS_CACHE:
 
-        patterns = Inventory.split_host_pattern(pattern)
-        hosts = self._evaluate_patterns(patterns)
+            patterns = Inventory.split_host_pattern(pattern)
+            hosts = self._evaluate_patterns(patterns)
 
-        # mainly useful for hostvars[host] access
-        if not ignore_limits_and_restrictions:
-            # exclude hosts not in a subset, if defined
-            if self._subset:
-                subset = self._evaluate_patterns(self._subset)
-                hosts = [ h for h in hosts if h in subset ]
+            # mainly useful for hostvars[host] access
+            if not ignore_limits_and_restrictions:
+                # exclude hosts not in a subset, if defined
+                if self._subset:
+                    subset = self._evaluate_patterns(self._subset)
+                    hosts = [ h for h in hosts if h in subset ]
 
-            # exclude hosts mentioned in any restriction (ex: failed hosts)
-            if self._restriction is not None:
-                hosts = [ h for h in hosts if h in self._restriction ]
+                # exclude hosts mentioned in any restriction (ex: failed hosts)
+                if self._restriction is not None:
+                    hosts = [ h for h in hosts if h in self._restriction ]
 
-        HOSTS_PATTERNS_CACHE[pattern_hash] = hosts[:]
-        return hosts
+            seen = set()
+            HOSTS_PATTERNS_CACHE[pattern_hash] = [x for x in hosts if x not in seen and not seen.add(x)]
+
+        return HOSTS_PATTERNS_CACHE[pattern_hash][:]
 
     @classmethod
     def split_host_pattern(cls, pattern):
@@ -227,15 +239,13 @@ class Inventory(object):
         # If it doesn't, it could still be a single pattern. This accounts for
         # non-separator uses of colons: IPv6 addresses and [x:y] host ranges.
         else:
-            (base, port) = parse_address(pattern, allow_ranges=True)
-            if base:
+            try:
+                (base, port) = parse_address(pattern, allow_ranges=True)
                 patterns = [pattern]
-
-            # The only other case we accept is a ':'-separated list of patterns.
-            # This mishandles IPv6 addresses, and is retained only for backwards
-            # compatibility.
-
-            else:
+            except:
+                # The only other case we accept is a ':'-separated list of patterns.
+                # This mishandles IPv6 addresses, and is retained only for backwards
+                # compatibility.
                 patterns = re.findall(
                     r'''(?:             # We want to match something comprising:
                             [^\s:\[\]]  # (anything other than whitespace or ':[]'
@@ -388,7 +398,7 @@ class Inventory(object):
                     end = -1
                 subscript = (int(start), int(end))
                 if sep == '-':
-                    display.deprecated("Use [x:y] inclusive subscripts instead of [x-y]", version=2.0, removed=True)
+                    display.warning("Use [x:y] inclusive subscripts instead of [x-y] which has been removed")
 
         return (pattern, subscript)
 
@@ -731,12 +741,12 @@ class Inventory(object):
 
             if group and host is None:
                 # load vars in dir/group_vars/name_of_group
-                base_path = os.path.realpath(os.path.join(basedir, "group_vars/%s" % group.name))
-                results = self._variable_manager.add_group_vars_file(base_path, self._loader)
+                base_path = os.path.abspath(os.path.join(to_unicode(basedir, errors='strict'), "group_vars/%s" % group.name))
+                results = combine_vars(results, self._variable_manager.add_group_vars_file(base_path, self._loader))
             elif host and group is None:
                 # same for hostvars in dir/host_vars/name_of_host
-                base_path = os.path.realpath(os.path.join(basedir, "host_vars/%s" % host.name))
-                results = self._variable_manager.add_host_vars_file(base_path, self._loader)
+                base_path = os.path.abspath(os.path.join(to_unicode(basedir, errors='strict'), "host_vars/%s" % host.name))
+                results = combine_vars(results, self._variable_manager.add_host_vars_file(base_path, self._loader))
 
         # all done, results is a dictionary of variables for this particular host.
         return results

lib/ansible/inventory/dir.py

@@ -205,7 +205,7 @@ class InventoryDirectory(object):
         # because the __eq__/__ne__ methods in Host() compare the
         # name fields rather than references, we use id() here to
         # do the object comparison for merges
-        if id(self.hosts[host.name]) != id(host):
+        if self.hosts[host.name] != host:
             # different object, merge
             self._merge_hosts(self.hosts[host.name], host)
 

lib/ansible/inventory/host.py

@@ -19,6 +19,8 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import uuid
+
 from ansible.inventory.group import Group
 from ansible.utils.vars import combine_vars
 
@@ -38,7 +40,7 @@ class Host:
     def __eq__(self, other):
         if not isinstance(other, Host):
             return False
-        return self.name == other.name
+        return self._uuid == other._uuid
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -55,6 +57,7 @@ class Host:
             name=self.name,
             vars=self.vars.copy(),
             address=self.address,
+            uuid=self._uuid,
             gathered_facts=self._gathered_facts,
             groups=groups,
         )
@@ -65,6 +68,7 @@ class Host:
         self.name    = data.get('name')
         self.vars    = data.get('vars', dict())
         self.address = data.get('address', '')
+        self._uuid   = data.get('uuid', uuid.uuid4())
 
         groups = data.get('groups', [])
         for group_data in groups:
@@ -84,6 +88,7 @@ class Host:
             self.set_variable('ansible_port', int(port))
 
         self._gathered_facts = False
+        self._uuid = uuid.uuid4()
 
     def __repr__(self):
         return self.get_name()

lib/ansible/inventory/ini.py

@@ -124,6 +124,9 @@ class InventoryParser(object):
                     del pending_declarations[groupname]
 
                 continue
+            elif line.startswith('['):
+                self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + \
+                                  "in the section entry, and that there are no other invalid characters")
 
             # It's not a section, so the current state tells us what kind of
             # definition it must be. The individual parsers will raise an
@@ -264,9 +267,12 @@ class InventoryParser(object):
         # Can the given hostpattern be parsed as a host with an optional port
         # specification?
 
-        (pattern, port) = parse_address(hostpattern, allow_ranges=True)
-        if not pattern:
-            self._raise_error("Can't parse '%s' as host[:port]" % hostpattern)
+        try:
+            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
+        except:
+            # not a recognizable host pattern
+            pattern = hostpattern
+            port = None
 
         # Once we have separated the pattern, we expand it into list of one or
         # more hostnames, depending on whether it contains any [x:y] ranges.

lib/ansible/inventory/script.py

@@ -31,6 +31,7 @@ from ansible.errors import AnsibleError
 from ansible.inventory.host import Host
 from ansible.inventory.group import Group
 from ansible.module_utils.basic import json_dict_bytes_to_unicode
+from ansible.utils.unicode import to_str, to_unicode
 
 
 class InventoryScript:
@@ -57,12 +58,17 @@ class InventoryScript:
         if sp.returncode != 0:
             raise AnsibleError("Inventory script (%s) had an execution error: %s " % (filename,stderr))
 
-        self.data = stdout
+        # make sure script output is unicode so that json loader will output
+        # unicode strings itself
+        try:
+            self.data = to_unicode(stdout, errors="strict")
+        except Exception as e:
+            raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_str(self.filename), to_str(e)))
+
         # see comment about _meta below
         self.host_vars_from_top = None
         self._parse(stderr)
 
-
     def _parse(self, err):
 
         all_hosts = {}
@@ -72,13 +78,11 @@ class InventoryScript:
             self.raw = self._loader.load(self.data)
         except Exception as e:
             sys.stderr.write(err + "\n")
-            raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(self.filename, str(e)))
+            raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename), to_str(e)))
 
         if not isinstance(self.raw, Mapping):
             sys.stderr.write(err + "\n")
-            raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(self.filename))
-
-        self.raw  = json_dict_bytes_to_unicode(self.raw)
+            raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename)))
 
         group = None
         for (group_name, data) in self.raw.items():
@@ -103,7 +107,7 @@ class InventoryScript:
             if not isinstance(data, dict):
                 data = {'hosts': data}
             # is not those subkeys, then simplified syntax, host with vars
-            elif not any(k in data for k in ('hosts','vars')):
+            elif not any(k in data for k in ('hosts','vars','children')):
                 data = {'hosts': [group_name], 'vars': data}
 
             if 'hosts' in data:
@@ -112,7 +116,7 @@ class InventoryScript:
                         "data for the host list:\n %s" % (group_name, data))
 
                 for hostname in data['hosts']:
-                    if not hostname in all_hosts:
+                    if hostname not in all_hosts:
                         all_hosts[hostname] = Host(hostname)
                     host = all_hosts[hostname]
                     group.add_host(host)
@@ -145,10 +149,12 @@ class InventoryScript:
     def get_host_variables(self, host):
         """ Runs <script> --host <hostname> to determine additional host variables """
         if self.host_vars_from_top is not None:
-            got = self.host_vars_from_top.get(host.name, {})
+            try:
+                got = self.host_vars_from_top.get(host.name, {})
+            except AttributeError as e:
+                raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_str(e)))
             return got
 
-
         cmd = [self.filename, "--host", host.name]
         try:
             sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -161,4 +167,3 @@ class InventoryScript:
             return json_dict_bytes_to_unicode(self._loader.load(out))
         except ValueError:
             raise AnsibleError("could not parse post variable response: %s, %s" % (cmd, out))
-

lib/ansible/module_utils/api.py

@@ -0,0 +1,103 @@
+#
+# (c) 2015 Brian Ccoa, <bcoca@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+This module adds shared support for generic api modules
+
+In order to use this module, include it as part of a custom
+module as shown below.
+
+** Note: The order of the import statements does matter. **
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.api import *
+
+The 'api' module provides the following common argument specs:
+
+    * rate limit spec
+        - rate: number of requests per time unit (int)
+        - rate_limit: time window in which the limit is applied in seconds
+
+    * retry spec
+        - retries: number of attempts
+        - retry_pause: delay between attempts in seconds
+
+"""
+import time
+
+def rate_limit_argument_spec(spec=None):
+    """Creates an argument spec for working with rate limiting"""
+    arg_spec = (dict(
+        rate=dict(type='int'),
+        rate_limit=dict(type='int'),
+    ))
+    if spec:
+        arg_spec.update(spec)
+    return arg_spec
+
+def retry_argument_spec(spec=None):
+    """Creates an argument spec for working with retrying"""
+    arg_spec = (dict(
+        retries=dict(type='int'),
+        retry_pause=dict(type='float', default=1),
+    ))
+    if spec:
+        arg_spec.update(spec)
+    return arg_spec
+
+def rate_limit(rate=None, rate_limit=None):
+    """rate limiting decorator"""
+    minrate = None
+    if rate is not None and rate_limit is not None:
+        minrate = float(rate_limit) / float(rate)
+    def wrapper(f):
+        last = [0.0]
+        def ratelimited(*args,**kwargs):
+            if minrate is not None:
+                elapsed = time.clock() - last[0]
+                left = minrate - elapsed
+                if left > 0:
+                    time.sleep(left)
+                last[0] = time.clock()
+            ret = f(*args,**kwargs)
+            return ret
+        return ratelimited
+    return wrapper
+
+def retry(retries=None, retry_pause=1):
+    """Retry decorator"""
+    def wrapper(f):
+        retry_count = 0
+        def retried(*args,**kwargs):
+            if retries is not None:
+                ret = None
+                while True:
+                    retry_count += 1
+                    if retry_count >= retries:
+                        raise Exception("Retry limit exceeded: %d" % retries)
+                    try:
+                        ret = f(*args,**kwargs)
+                    except:
+                        pass
+                    if ret:
+                        break
+                    time.sleep(retry_pause)
+                return ret
+        return retried
+    return wrapper
+

lib/ansible/module_utils/basic.py

@@ -3,27 +3,27 @@
 # Modules you write using this snippet, which is embedded dynamically by Ansible
 # still belong to the author of the module, and may assign their own license
 # to the complete work.
-# 
+#
 # Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
 # All rights reserved.
 #
-# Redistribution and use in source and binary forms, with or without modification, 
+# Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
 #
-#    * Redistributions of source code must retain the above copyright 
+#    * Redistributions of source code must retain the above copyright
 #      notice, this list of conditions and the following disclaimer.
-#    * Redistributions in binary form must reproduce the above copyright notice, 
-#      this list of conditions and the following disclaimer in the documentation 
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
 #      and/or other materials provided with the distribution.
 #
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
-# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
-# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
@@ -34,8 +34,8 @@ ANSIBLE_VERSION = "<<ANSIBLE_VERSION>>"
 MODULE_ARGS = "<<INCLUDE_ANSIBLE_MODULE_ARGS>>"
 MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
 
-BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1]
-BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0]
+BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1, True]
+BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0, False]
 BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE
 
 SELINUX_SPECIAL_FS="<<SELINUX_SPECIAL_FILESYSTEMS>>"
@@ -65,6 +65,7 @@ import grp
 import pwd
 import platform
 import errno
+import datetime
 from itertools import repeat, chain
 
 try:
@@ -112,6 +113,12 @@ else:
     def iteritems(d):
         return d.iteritems()
 
+try:
+    reduce
+except NameError:
+    # Python 3
+    from functools import reduce
+
 try:
     NUMBERTYPES = (int, long, float)
 except NameError:
@@ -184,15 +191,15 @@ except ImportError:
         pass
 
 try:
-    from ast import literal_eval as _literal_eval
+    from ast import literal_eval
 except ImportError:
-    # a replacement for literal_eval that works with python 2.4. from: 
+    # a replacement for literal_eval that works with python 2.4. from:
     # https://mail.python.org/pipermail/python-list/2009-September/551880.html
     # which is essentially a cut/paste from an earlier (2.6) version of python's
     # ast.py
     from compiler import ast, parse
 
-    def _literal_eval(node_or_string):
+    def literal_eval(node_or_string):
         """
         Safely evaluate an expression node or a string containing a Python
         expression.  The string or node provided may only consist of the  following
@@ -222,10 +229,11 @@ except ImportError:
             raise ValueError('malformed string')
         return _convert(node_or_string)
 
+_literal_eval = literal_eval
 
 FILE_COMMON_ARGUMENTS=dict(
     src = dict(),
-    mode = dict(),
+    mode = dict(type='raw'),
     owner = dict(),
     group = dict(),
     seuser = dict(),
@@ -423,10 +431,13 @@ def remove_values(value, no_log_strings):
         for omit_me in no_log_strings:
             if omit_me in stringy_value:
                 return 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
+    elif isinstance(value, datetime.datetime):
+        value = value.isoformat()
     else:
         raise TypeError('Value of unknown type: %s, %s' % (type(value), value))
     return value
 
+
 def heuristic_log_sanitize(data, no_log_values=None):
     ''' Remove strings that look like passwords from log messages '''
     # Currently filters:
@@ -514,8 +525,14 @@ class AnsibleModule(object):
         self.no_log = no_log
         self.cleanup_files = []
         self._debug = False
+        self._diff = False
+        self._verbosity = 0
+        # May be used to set modifications to the environment for any
+        # run_command invocation
+        self.run_command_environ_update = {}
 
         self.aliases = {}
+        self._legal_inputs = ['_ansible_check_mode', '_ansible_no_log', '_ansible_debug', '_ansible_diff', '_ansible_verbosity']
 
         if add_file_common_args:
             for k, v in FILE_COMMON_ARGUMENTS.items():
@@ -524,6 +541,15 @@ class AnsibleModule(object):
 
         self.params = self._load_params()
 
+        # append to legal_inputs and then possibly check against them
+        try:
+            self.aliases = self._handle_aliases()
+        except Exception:
+            e = get_exception()
+            # Use exceptions here because it isn't safe to call fail_json until no_log is processed
+            print('{"failed": true, "msg": "Module alias error: %s"}' % str(e))
+            sys.exit(1)
+
         # Save parameter values that should never be logged
         self.no_log_values = set()
         # Use the argspec to determine which args are no_log
@@ -534,15 +560,10 @@ class AnsibleModule(object):
                 if no_log_object:
                     self.no_log_values.update(return_values(no_log_object))
 
-        # check the locale as set by the current environment, and
-        # reset to LANG=C if it's an invalid/unavailable locale
+        # check the locale as set by the current environment, and reset to
+        # a known valid (LANG=C) if it's an invalid/unavailable locale
         self._check_locale()
 
-        self._legal_inputs = ['_ansible_check_mode', '_ansible_no_log', '_ansible_debug']
-
-        # append to legal_inputs and then possibly check against them
-        self.aliases = self._handle_aliases()
-
         self._check_arguments(check_invalid_arguments)
 
         # check exclusive early
@@ -560,6 +581,7 @@ class AnsibleModule(object):
                 'int': self._check_type_int,
                 'float': self._check_type_float,
                 'path': self._check_type_path,
+                'raw': self._check_type_raw,
             }
         if not bypass_checks:
             self._check_required_arguments()
@@ -571,7 +593,7 @@ class AnsibleModule(object):
 
         self._set_defaults(pre=False)
 
-        if not self.no_log:
+        if not self.no_log and self._verbosity >= 3:
             self._log_invocation()
 
         # finally, make sure we're in a sane working dir
@@ -745,7 +767,7 @@ class AnsibleModule(object):
         context = self.selinux_default_context(path)
         return self.set_context_if_different(path, context, False)
 
-    def set_context_if_different(self, path, context, changed):
+    def set_context_if_different(self, path, context, changed, diff=None):
 
         if not HAVE_SELINUX or not self.selinux_enabled():
             return changed
@@ -766,6 +788,14 @@ class AnsibleModule(object):
                         new_context[i] = cur_context[i]
 
         if cur_context != new_context:
+            if diff is not None:
+                if 'before' not in diff:
+                    diff['before'] = {}
+                diff['before']['secontext'] = cur_context
+                if 'after' not in diff:
+                    diff['after'] = {}
+                diff['after']['secontext'] = new_context
+
             try:
                 if self.check_mode:
                     return True
@@ -779,7 +809,7 @@ class AnsibleModule(object):
             changed = True
         return changed
 
-    def set_owner_if_different(self, path, owner, changed):
+    def set_owner_if_different(self, path, owner, changed, diff=None):
         path = os.path.expanduser(path)
         if owner is None:
             return changed
@@ -792,6 +822,15 @@ class AnsibleModule(object):
             except KeyError:
                 self.fail_json(path=path, msg='chown failed: failed to look up user %s' % owner)
         if orig_uid != uid:
+
+            if diff is not None:
+                if 'before' not in diff:
+                    diff['before'] = {}
+                diff['before']['owner'] = orig_uid
+                if 'after' not in diff:
+                    diff['after'] = {}
+                diff['after']['owner'] = uid
+
             if self.check_mode:
                 return True
             try:
@@ -801,7 +840,7 @@ class AnsibleModule(object):
             changed = True
         return changed
 
-    def set_group_if_different(self, path, group, changed):
+    def set_group_if_different(self, path, group, changed, diff=None):
         path = os.path.expanduser(path)
         if group is None:
             return changed
@@ -814,6 +853,15 @@ class AnsibleModule(object):
             except KeyError:
                 self.fail_json(path=path, msg='chgrp failed: failed to look up group %s' % group)
         if orig_gid != gid:
+
+            if diff is not None:
+                if 'before' not in diff:
+                    diff['before'] = {}
+                diff['before']['group'] = orig_gid
+                if 'after' not in diff:
+                    diff['after'] = {}
+                diff['after']['group'] = gid
+
             if self.check_mode:
                 return True
             try:
@@ -823,7 +871,7 @@ class AnsibleModule(object):
             changed = True
         return changed
 
-    def set_mode_if_different(self, path, mode, changed):
+    def set_mode_if_different(self, path, mode, changed, diff=None):
         path = os.path.expanduser(path)
         path_stat = os.lstat(path)
 
@@ -842,9 +890,22 @@ class AnsibleModule(object):
                                    msg="mode must be in octal or symbolic form",
                                    details=str(e))
 
+                if mode != stat.S_IMODE(mode):
+                    # prevent mode from having extra info orbeing invalid long number
+                    self.fail_json(path=path, msg="Invalid mode supplied, only permission info is allowed", details=mode)
+
         prev_mode = stat.S_IMODE(path_stat.st_mode)
 
         if prev_mode != mode:
+
+            if diff is not None:
+                if 'before' not in diff:
+                    diff['before'] = {}
+                diff['before']['mode'] = oct(prev_mode)
+                if 'after' not in diff:
+                    diff['after'] = {}
+                diff['after']['mode'] = oct(mode)
+
             if self.check_mode:
                 return True
             # FIXME: comparison against string above will cause this to be executed
@@ -886,7 +947,7 @@ class AnsibleModule(object):
     def _symbolic_mode_to_octal(self, path_stat, symbolic_mode):
         new_mode = stat.S_IMODE(path_stat.st_mode)
 
-        mode_re = re.compile(r'^(?P<users>[ugoa]+)(?P<operator>[-+=])(?P<perms>[rwxXst]*|[ugo])$')
+        mode_re = re.compile(r'^(?P<users>[ugoa]+)(?P<operator>[-+=])(?P<perms>[rwxXst-]*|[ugo])$')
         for mode in symbolic_mode.split(','):
             match = mode_re.match(mode)
             if match:
@@ -903,14 +964,14 @@ class AnsibleModule(object):
             else:
                 raise ValueError("bad symbolic permission for mode: %s" % mode)
         return new_mode
-    
+
     def _apply_operation_to_mode(self, user, operator, mode_to_apply, current_mode):
         if operator  ==  '=':
             if user == 'u': mask = stat.S_IRWXU | stat.S_ISUID
             elif user == 'g': mask = stat.S_IRWXG | stat.S_ISGID
             elif user == 'o': mask = stat.S_IRWXO | stat.S_ISVTX
-            
-            # mask out u, g, or o permissions from current_mode and apply new permissions   
+
+            # mask out u, g, or o permissions from current_mode and apply new permissions
             inverse_mask = mask ^ PERM_BITS
             new_mode = (current_mode & inverse_mask) | mode_to_apply
         elif operator == '+':
@@ -918,10 +979,10 @@ class AnsibleModule(object):
         elif operator == '-':
             new_mode = current_mode - (current_mode & mode_to_apply)
         return new_mode
-        
+
     def _get_octal_mode_from_symbolic_perms(self, path_stat, user, perms):
         prev_mode = stat.S_IMODE(path_stat.st_mode)
-        
+
         is_directory = stat.S_ISDIR(path_stat.st_mode)
         has_x_permissions = (prev_mode & EXEC_PERM_BITS) > 0
         apply_X_permission = is_directory or has_x_permissions
@@ -978,27 +1039,27 @@ class AnsibleModule(object):
         or_reduce = lambda mode, perm: mode | user_perms_to_modes[user][perm]
         return reduce(or_reduce, perms, 0)
 
-    def set_fs_attributes_if_different(self, file_args, changed):
+    def set_fs_attributes_if_different(self, file_args, changed, diff=None):
         # set modes owners and context as needed
         changed = self.set_context_if_different(
-            file_args['path'], file_args['secontext'], changed
+            file_args['path'], file_args['secontext'], changed, diff
         )
         changed = self.set_owner_if_different(
-            file_args['path'], file_args['owner'], changed
+            file_args['path'], file_args['owner'], changed, diff
         )
         changed = self.set_group_if_different(
-            file_args['path'], file_args['group'], changed
+            file_args['path'], file_args['group'], changed, diff
         )
         changed = self.set_mode_if_different(
-            file_args['path'], file_args['mode'], changed
+            file_args['path'], file_args['mode'], changed, diff
         )
         return changed
 
-    def set_directory_attributes_if_different(self, file_args, changed):
-        return self.set_fs_attributes_if_different(file_args, changed)
+    def set_directory_attributes_if_different(self, file_args, changed, diff=None):
+        return self.set_fs_attributes_if_different(file_args, changed, diff)
 
-    def set_file_attributes_if_different(self, file_args, changed):
-        return self.set_fs_attributes_if_different(file_args, changed)
+    def set_file_attributes_if_different(self, file_args, changed, diff=None):
+        return self.set_fs_attributes_if_different(file_args, changed, diff)
 
     def add_path_info(self, kwargs):
         '''
@@ -1051,7 +1112,6 @@ class AnsibleModule(object):
             # as it would be returned by locale.getdefaultlocale()
             locale.setlocale(locale.LC_ALL, '')
         except locale.Error:
-            e = get_exception()
             # fallback to the 'C' locale, which may cause unicode
             # issues but is preferable to simply failing because
             # of an unknown locale
@@ -1064,6 +1124,7 @@ class AnsibleModule(object):
             self.fail_json(msg="An unknown error was encountered while attempting to validate the locale: %s" % e)
 
     def _handle_aliases(self):
+        # this uses exceptions as it happens before we can safely call fail_json
         aliases_results = {} #alias:canon
         for (k,v) in self.argument_spec.items():
             self._legal_inputs.append(k)
@@ -1072,11 +1133,11 @@ class AnsibleModule(object):
             required = v.get('required', False)
             if default is not None and required:
                 # not alias specific but this is a good place to check this
-                self.fail_json(msg="internal error: required and default are mutually exclusive for %s" % k)
+                raise Exception("internal error: required and default are mutually exclusive for %s" % k)
             if aliases is None:
                 continue
             if type(aliases) != list:
-                self.fail_json(msg='internal error: aliases must be a list')
+                raise Exception('internal error: aliases must be a list')
             for alias in aliases:
                 self._legal_inputs.append(alias)
                 aliases_results[alias] = k
@@ -1099,9 +1160,19 @@ class AnsibleModule(object):
             elif k == '_ansible_debug':
                 self._debug = self.boolean(v)
 
+            elif k == '_ansible_diff':
+                self._diff = self.boolean(v)
+
+            elif k == '_ansible_verbosity':
+                self._verbosity = v
+
             elif check_invalid_arguments and k not in self._legal_inputs:
                 self.fail_json(msg="unsupported parameter for module: %s" % k)
 
+            #clean up internal params:
+            if k.startswith('_ansible_'):
+                del self.params[k]
+
     def _count_terms(self, check):
         count = 0
         for term in check:
@@ -1192,11 +1263,7 @@ class AnsibleModule(object):
                 return (str, None)
             return str
         try:
-            result = None
-            if not locals:
-                result = _literal_eval(str)
-            else:
-                result = _literal_eval(str, None, locals)
+            result = literal_eval(str)
             if include_exceptions:
                 return (result, None)
             else:
@@ -1274,7 +1341,7 @@ class AnsibleModule(object):
         if isinstance(value, bool):
             return value
 
-        if isinstance(value, basestring):
+        if isinstance(value, basestring) or isinstance(value, int):
             return self.boolean(value)
 
         raise TypeError('%s cannot be converted to a bool' % type(value))
@@ -1301,15 +1368,23 @@ class AnsibleModule(object):
         value = self._check_type_str(value)
         return os.path.expanduser(os.path.expandvars(value))
 
+    def _check_type_raw(self, value):
+        return value
+
 
     def _check_argument_types(self):
         ''' ensure all arguments have the requested type '''
         for (k, v) in self.argument_spec.items():
             wanted = v.get('type', None)
-            if wanted is None:
-                continue
             if k not in self.params:
                 continue
+            if wanted is None:
+                # Mostly we want to default to str.
+                # For values set to None explicitly, return None instead as
+                # that allows a user to unset a parameter
+                if self.params[k] is None:
+                    continue
+                wanted = 'str'
 
             value = self.params[k]
 
@@ -1422,16 +1497,15 @@ class AnsibleModule(object):
                 arg_val = str(arg_val)
             elif isinstance(arg_val, unicode):
                 arg_val = arg_val.encode('utf-8')
-            msg.append('%s=%s ' % (arg, arg_val))
+            msg.append('%s=%s' % (arg, arg_val))
         if msg:
-            msg = 'Invoked with %s' % ''.join(msg)
+            msg = 'Invoked with %s' % ' '.join(msg)
         else:
             msg = 'Invoked'
 
         self.log(msg, log_args=log_args)
 
 
-
     def _set_cwd(self):
         try:
             cwd = os.getcwd()
@@ -1439,7 +1513,7 @@ class AnsibleModule(object):
                 raise
             return cwd
         except:
-            # we don't have access to the cwd, probably because of sudo. 
+            # we don't have access to the cwd, probably because of sudo.
             # Try and move to a neutral location to prevent errors
             for cwd in [os.path.expandvars('$HOME'), tempfile.gettempdir()]:
                 try:
@@ -1448,9 +1522,9 @@ class AnsibleModule(object):
                         return cwd
                 except:
                     pass
-        # we won't error here, as it may *not* be a problem, 
+        # we won't error here, as it may *not* be a problem,
         # and we don't want to break modules unnecessarily
-        return None    
+        return None
 
     def get_bin_path(self, arg, required=False, opt_dirs=[]):
         '''
@@ -1472,6 +1546,8 @@ class AnsibleModule(object):
             if p not in paths and os.path.exists(p):
                 paths.append(p)
         for d in paths:
+            if not d:
+                continue
             path = os.path.join(d, arg)
             if os.path.exists(path) and is_executable(path):
                 bin_path = path
@@ -1524,6 +1600,8 @@ class AnsibleModule(object):
         self.add_path_info(kwargs)
         if not 'changed' in kwargs:
             kwargs['changed'] = False
+        if 'invocation' not in kwargs:
+            kwargs['invocation'] = {'module_args': self.params}
         kwargs = remove_values(kwargs, self.no_log_values)
         self.do_cleanup_files()
         print(self.jsonify(kwargs))
@@ -1534,11 +1612,26 @@ class AnsibleModule(object):
         self.add_path_info(kwargs)
         assert 'msg' in kwargs, "implementation error -- msg to explain the error is required"
         kwargs['failed'] = True
+        if 'invocation' not in kwargs:
+            kwargs['invocation'] = {'module_args': self.params}
         kwargs = remove_values(kwargs, self.no_log_values)
         self.do_cleanup_files()
         print(self.jsonify(kwargs))
         sys.exit(1)
 
+    def fail_on_missing_params(self, required_params=None):
+        ''' This is for checking for required params when we can not check via argspec because we
+        need more information than is simply given in the argspec.
+        '''
+        if not required_params:
+            return
+        missing_params = []
+        for required_param in required_params:
+            if not self.params.get(required_param):
+                missing_params.append(required_param)
+        if missing_params:
+            self.fail_json(msg="missing required arguments: %s" % ','.join(missing_params))
+
     def digest_from_file(self, filename, algorithm):
         ''' Return hex digest of local file for a digest_method specified by name, or None if file is not present. '''
         if not os.path.exists(filename):
@@ -1613,7 +1706,7 @@ class AnsibleModule(object):
                 e = get_exception()
                 sys.stderr.write("could not cleanup %s: %s" % (tmpfile, e))
 
-    def atomic_move(self, src, dest):
+    def atomic_move(self, src, dest, unsafe_writes=False):
         '''atomically move src to dest, copying attributes from dest, returns true on success
         it uses os.rename to ensure this as it is an atomic operation, rest of the function is
         to work around limitations, corner cases and ensure selinux context is saved if possible'''
@@ -1653,43 +1746,63 @@ class AnsibleModule(object):
             os.rename(src, dest)
         except (IOError, OSError):
             e = get_exception()
-            # only try workarounds for errno 18 (cross device), 1 (not permitted),  13 (permission denied)
-            # and 26 (text file busy) which happens on vagrant synced folders
             if e.errno not in [errno.EPERM, errno.EXDEV, errno.EACCES, errno.ETXTBSY]:
+                # only try workarounds for errno 18 (cross device), 1 (not permitted),  13 (permission denied)
+                # and 26 (text file busy) which happens on vagrant synced folders and other 'exotic' non posix file systems
                 self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e))
-
-            dest_dir = os.path.dirname(dest)
-            dest_file = os.path.basename(dest)
-            try:
-                tmp_dest = tempfile.NamedTemporaryFile(
-                    prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file)
-            except (OSError, IOError):
-                e = get_exception()
-                self.fail_json(msg='The destination directory (%s) is not writable by the current user.' % dest_dir)
-
-            try: # leaves tmp file behind when sudo and  not root
-                if switched_user and os.getuid() != 0:
-                    # cleanup will happen by 'rm' of tempdir
-                    # copy2 will preserve some metadata
-                    shutil.copy2(src, tmp_dest.name)
-                else:
-                    shutil.move(src, tmp_dest.name)
-                if self.selinux_enabled():
-                    self.set_context_if_different(
-                        tmp_dest.name, context, False)
+            else:
+                dest_dir = os.path.dirname(dest)
+                dest_file = os.path.basename(dest)
                 try:
-                    tmp_stat = os.stat(tmp_dest.name)
-                    if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid):
-                        os.chown(tmp_dest.name, dest_stat.st_uid, dest_stat.st_gid)
-                except OSError:
+                    tmp_dest = tempfile.NamedTemporaryFile(
+                        prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file)
+                except (OSError, IOError):
                     e = get_exception()
-                    if e.errno != errno.EPERM:
-                        raise
-                os.rename(tmp_dest.name, dest)
-            except (shutil.Error, OSError, IOError):
-                e = get_exception()
-                self.cleanup(tmp_dest.name)
-                self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e))
+                    self.fail_json(msg='The destination directory (%s) is not writable by the current user. Error was: %s' % (dest_dir, e))
+
+                try: # leaves tmp file behind when sudo and  not root
+                    if switched_user and os.getuid() != 0:
+                        # cleanup will happen by 'rm' of tempdir
+                        # copy2 will preserve some metadata
+                        shutil.copy2(src, tmp_dest.name)
+                    else:
+                        shutil.move(src, tmp_dest.name)
+                    if self.selinux_enabled():
+                        self.set_context_if_different(
+                            tmp_dest.name, context, False)
+                    try:
+                        tmp_stat = os.stat(tmp_dest.name)
+                        if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid):
+                            os.chown(tmp_dest.name, dest_stat.st_uid, dest_stat.st_gid)
+                    except OSError:
+                        e = get_exception()
+                        if e.errno != errno.EPERM:
+                            raise
+                    os.rename(tmp_dest.name, dest)
+                except (shutil.Error, OSError, IOError):
+                    e = get_exception()
+                    # sadly there are some situations where we cannot ensure atomicity, but only if
+                    # the user insists and we get the appropriate error we update the file unsafely
+                    if unsafe_writes and e.errno == errno.EBUSY:
+                        #TODO: issue warning that this is an unsafe operation, but doing it cause user insists
+                        try:
+                            try:
+                                out_dest = open(dest, 'wb')
+                                in_src = open(src, 'rb')
+                                shutil.copyfileobj(in_src, out_dest)
+                            finally: # assuring closed files in 2.4 compatible way
+                                if out_dest:
+                                    out_dest.close()
+                                if in_src:
+                                    in_src.close()
+                        except (shutil.Error, OSError, IOError):
+                            e = get_exception()
+                            self.fail_json(msg='Could not write data to file (%s) from (%s): %s' % (dest, src, e))
+
+                    else:
+                        self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e))
+
+                    self.cleanup(tmp_dest.name)
 
         if creating:
             # make sure the file has the correct permissions
@@ -1704,25 +1817,29 @@ class AnsibleModule(object):
             # rename might not preserve context
             self.set_context_if_different(dest, context, False)
 
-    def run_command(self, args, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None):
+    def run_command(self, args, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None, environ_update=None):
         '''
         Execute a command, returns rc, stdout, and stderr.
-        args is the command to run
-        If args is a list, the command will be run with shell=False.
-        If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False
-        If args is a string and use_unsafe_shell=True it run with shell=True.
-        Other arguments:
-        - check_rc (boolean)    Whether to call fail_json in case of
-                                non zero RC.  Default is False.
-        - close_fds (boolean)   See documentation for subprocess.Popen().
-                                Default is True.
-        - executable (string)   See documentation for subprocess.Popen().
-                                Default is None.
-        - prompt_regex (string) A regex string (not a compiled regex) which
-                                can be used to detect prompts in the stdout
-                                which would otherwise cause the execution
-                                to hang (especially if no input data is
-                                specified)
+
+        :arg args: is the command to run
+            * If args is a list, the command will be run with shell=False.
+            * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False
+            * If args is a string and use_unsafe_shell=True it runs with shell=True.
+        :kw check_rc: Whether to call fail_json in case of non zero RC.
+            Default False
+        :kw close_fds: See documentation for subprocess.Popen(). Default True
+        :kw executable: See documentation for subprocess.Popen(). Default None
+        :kw data: If given, information to write to the stdin of the command
+        :kw binary_data: If False, append a newline to the data.  Default False
+        :kw path_prefix: If given, additional path to find the command in.
+            This adds to the PATH environment vairable so helper commands in
+            the same directory can also be found
+        :kw cwd: iIf given, working directory to run the command inside
+        :kw use_unsafe_shell: See `args` parameter.  Default False
+        :kw prompt_regex: Regex string (not a compiled regex) which can be
+            used to detect prompts in the stdout which would otherwise cause
+            the execution to hang (especially if no input data is specified)
+        :kwarg environ_update: dictionary to *update* os.environ with
         '''
 
         shell = False
@@ -1733,7 +1850,9 @@ class AnsibleModule(object):
         elif isinstance(args, basestring) and use_unsafe_shell:
             shell = True
         elif isinstance(args, basestring):
-            args = shlex.split(args.encode('utf-8'))
+            if isinstance(args, unicode):
+                args = args.encode('utf-8')
+            args = shlex.split(args)
         else:
             msg = "Argument 'args' to run_command must be list or string"
             self.fail_json(rc=257, cmd=args, msg=msg)
@@ -1747,16 +1866,25 @@ class AnsibleModule(object):
 
         # expand things like $HOME and ~
         if not shell:
-            args = [ os.path.expandvars(os.path.expanduser(x)) for x in args ]
+            args = [ os.path.expandvars(os.path.expanduser(x)) for x in args if x is not None ]
 
         rc = 0
         msg = None
         st_in = None
 
-        # Set a temporary env path if a prefix is passed
-        env=os.environ
+        # Manipulate the environ we'll send to the new process
+        old_env_vals = {}
+        # We can set this from both an attribute and per call
+        for key, val in self.run_command_environ_update.items():
+            old_env_vals[key] = os.environ.get(key, None)
+            os.environ[key] = val
+        if environ_update:
+            for key, val in environ_update.items():
+                old_env_vals[key] = os.environ.get(key, None)
+                os.environ[key] = val
         if path_prefix:
-            env['PATH']="%s:%s" % (path_prefix, env['PATH'])
+            old_env_vals['PATH'] = os.environ['PATH']
+            os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH'])
 
         # create a printable version of the command for use
         # in reporting later, which strips out things like
@@ -1798,11 +1926,10 @@ class AnsibleModule(object):
             close_fds=close_fds,
             stdin=st_in,
             stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE
+            stderr=subprocess.PIPE,
+            env=os.environ,
         )
 
-        if path_prefix:
-            kwargs['env'] = env
         if cwd and os.path.isdir(cwd):
             kwargs['cwd'] = cwd
 
@@ -1825,7 +1952,6 @@ class AnsibleModule(object):
                 else:
                     running = args
                 self.log('Executing: ' + running)
-
             cmd = subprocess.Popen(args, **kwargs)
 
             # the communication logic here is essentially taken from that
@@ -1881,6 +2007,13 @@ class AnsibleModule(object):
         except:
             self.fail_json(rc=257, msg=traceback.format_exc(), cmd=clean_args)
 
+        # Restore env settings
+        for key, val in old_env_vals.items():
+            if val is None:
+                del os.environ[key]
+            else:
+                os.environ[key] = val
+
         if rc != 0 and check_rc:
             msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values)
             self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg)

lib/ansible/module_utils/cloudstack.py

@@ -35,6 +35,18 @@ try:
 except ImportError:
     has_lib_cs = False
 
+CS_HYPERVISORS = [
+    "KVM", "kvm",
+    "VMware", "vmware",
+    "BareMetal", "baremetal",
+    "XenServer", "xenserver",
+    "LXC", "lxc",
+    "HyperV", "hyperv",
+    "UCS", "ucs",
+    "OVM", "ovm",
+    "Simulator", "simulator",
+    ]
+
 def cs_argument_spec():
     return dict(
         api_key = dict(default=None),
@@ -78,6 +90,10 @@ class AnsibleCloudStack(object):
         self.returns = {}
         # these values will be casted to int
         self.returns_to_int = {}
+        # these keys will be compared case sensitive in self.has_changed()
+        self.case_sensitive_keys = [
+            'id',
+        ]
 
         self.module = module
         self._connect()
@@ -138,16 +154,14 @@ class AnsibleCloudStack(object):
                 continue
 
             if key in current_dict:
-
-                # API returns string for int in some cases, just to make sure
-                if isinstance(value, int):
-                    current_dict[key] = int(current_dict[key])
-                elif isinstance(value, str):
-                    current_dict[key] = str(current_dict[key])
-
-                # Only need to detect a singe change, not every item
-                if value != current_dict[key]:
+                if self.case_sensitive_keys and key in self.case_sensitive_keys:
+                    if str(value) != str(current_dict[key]):
+                        return True
+                # Test for diff in case insensitive way
+                elif str(value).lower() != str(current_dict[key]).lower():
                     return True
+            else:
+                return True
         return False
 
 
@@ -218,7 +232,7 @@ class AnsibleCloudStack(object):
         vms = self.cs.listVirtualMachines(**args)
         if vms:
             for v in vms['virtualmachine']:
-                if vm in [ v['name'], v['displayname'], v['id'] ]:
+                if vm.lower() in [ v['name'].lower(), v['displayname'].lower(), v['id'] ]:
                     self.vm = v
                     return self._get_by_key(key, self.vm)
         self.module.fail_json(msg="Virtual machine '%s' not found" % vm)
@@ -238,7 +252,7 @@ class AnsibleCloudStack(object):
 
         if zones:
             for z in zones['zone']:
-                if zone in [ z['name'], z['id'] ]:
+                if zone.lower() in [ z['name'].lower(), z['id'] ]:
                     self.zone = z
                     return self._get_by_key(key, self.zone)
         self.module.fail_json(msg="zone '%s' not found" % zone)

lib/ansible/module_utils/eapi.py

@@ -1,174 +0,0 @@
-#
-# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
-"""
-This module adds shared support for Arista EOS devices using eAPI over
-HTTP/S transport.   It is built on module_utils/urls.py which is required
-for proper operation.
-
-In order to use this module, include it as part of a custom
-module as shown below.
-
-** Note: The order of the import statements does matter. **
-
-from ansible.module_utils.basic import *
-from ansible.module_utils.urls import *
-from ansible.module_utils.eapi import *
-
-The eapi module provides the following common argument spec:
-
-    * host (str) - The IPv4 address or FQDN of the network device
-    * port (str) - Overrides the default port to use for the HTTP/S
-        connection.  The default values are 80 for HTTP and
-        443 for HTTPS
-    * username (str) - The username to use to authenticate the HTTP/S
-        connection.
-    * password (str) - The password to use to authenticate the HTTP/S
-        connection.
-    * use_ssl (bool) - Specifies whether or not to use an encrypted (HTTPS)
-        connection or not.  The default value is False.
-    * enable_mode (bool) - Specifies whether or not to enter `enable` mode
-        prior to executing the command list.  The default value is True
-    * enable_password (str) - The password for entering `enable` mode
-        on the switch if configured.
-    * device (dict) - Used to send the entire set of connectin parameters
-        as a dict object.  This argument is mutually exclusive with the
-        host argument
-
-In order to communicate with Arista EOS devices, the eAPI feature
-must be enabled and configured on the device.
-
-"""
-EAPI_COMMON_ARGS = dict(
-    host=dict(),
-    port=dict(),
-    username=dict(),
-    password=dict(no_log=True),
-    use_ssl=dict(default=True, type='bool'),
-    enable_mode=dict(default=True, type='bool'),
-    enable_password=dict(no_log=True),
-    device=dict()
-)
-
-def eapi_module(**kwargs):
-    """Append the common args to the argument_spec
-    """
-    spec = kwargs.get('argument_spec') or dict()
-
-    argument_spec = url_argument_spec()
-    argument_spec.update(EAPI_COMMON_ARGS)
-    if kwargs.get('argument_spec'):
-        argument_spec.update(kwargs['argument_spec'])
-    kwargs['argument_spec'] = argument_spec
-
-    module = AnsibleModule(**kwargs)
-
-    device = module.params.get('device') or dict()
-    for key, value in device.iteritems():
-        if key in EAPI_COMMON_ARGS:
-            module.params[key] = value
-
-    params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS))
-    for key, value in params.iteritems():
-        if key != 'device':
-            module.params[key] = value
-
-    return module
-
-def eapi_url(params):
-    """Construct a valid Arista eAPI URL
-    """
-    if params['use_ssl']:
-        proto = 'https'
-    else:
-        proto = 'http'
-    host = params['host']
-    url = '{}://{}'.format(proto, host)
-    if params['port']:
-        url = '{}:{}'.format(url, params['port'])
-    return '{}/command-api'.format(url)
-
-def to_list(arg):
-    """Convert the argument to a list object
-    """
-    if isinstance(arg, (list, tuple)):
-        return list(arg)
-    elif arg is not None:
-        return [arg]
-    else:
-        return []
-
-def eapi_body(commands, encoding, reqid=None):
-    """Create a valid eAPI JSON-RPC request message
-    """
-    params = dict(version=1, cmds=to_list(commands), format=encoding)
-    return dict(jsonrpc='2.0', id=reqid, method='runCmds', params=params)
-
-def eapi_enable_mode(params):
-    """Build commands for entering `enable` mode on the switch
-    """
-    if params['enable_mode']:
-        passwd = params['enable_password']
-        if passwd:
-            return dict(cmd='enable', input=passwd)
-        else:
-            return 'enable'
-
-def eapi_command(module, commands, encoding='json'):
-    """Send an ordered list of commands to the device over eAPI
-    """
-    commands = to_list(commands)
-    url = eapi_url(module.params)
-
-    enable = eapi_enable_mode(module.params)
-    if enable:
-        commands.insert(0, enable)
-
-    data = eapi_body(commands, encoding)
-    data = module.jsonify(data)
-
-    headers = {'Content-Type': 'application/json-rpc'}
-
-    module.params['url_username'] = module.params['username']
-    module.params['url_password'] = module.params['password']
-
-    response, headers = fetch_url(module, url, data=data, headers=headers,
-                                  method='POST')
-
-    if headers['status'] != 200:
-        module.fail_json(**headers)
-
-    response = module.from_json(response.read())
-    if 'error' in response:
-        err = response['error']
-        module.fail_json(msg='json-rpc error', **err)
-
-    if enable:
-        response['result'].pop(0)
-
-    return response['result'], headers
-
-def eapi_configure(module, commands):
-    """Send configuration commands to the device over eAPI
-    """
-    commands.insert(0, 'configure')
-    response, headers = eapi_command(module, commands)
-    response.pop(0)
-    return response, headers
-
-

lib/ansible/module_utils/ec2.py

@@ -26,9 +26,11 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import os
+from time import sleep
 
 try:
     import boto3
+    import botocore
     HAS_BOTO3 = True
 except:
     HAS_BOTO3 = False
@@ -46,8 +48,6 @@ class AnsibleAWSError(Exception):
 
 def boto3_conn(module, conn_type=None, resource=None, region=None, endpoint=None, **params):
     profile = params.pop('profile_name', None)
-    params['aws_session_token'] = params.pop('security_token', None)
-    params['verify'] = params.pop('validate_certs', None)
 
     if conn_type not in ['both', 'resource', 'client']:
         module.fail_json(msg='There is an issue in the code of the module. You must specify either both, resource or client to the conn_type parameter in the boto3_conn function call')
@@ -138,10 +138,16 @@ def get_aws_connection_info(module, boto3=False):
         elif 'EC2_REGION' in os.environ:
             region = os.environ['EC2_REGION']
         else:
-            # boto.config.get returns None if config not found
-            region = boto.config.get('Boto', 'aws_region')
-            if not region:
-                region = boto.config.get('Boto', 'ec2_region')
+            if not boto3:
+                # boto.config.get returns None if config not found
+                region = boto.config.get('Boto', 'aws_region')
+                if not region:
+                    region = boto.config.get('Boto', 'ec2_region')
+            elif HAS_BOTO3:
+                # here we don't need to make an additional call, will default to 'us-east-1' if the below evaluates to None.
+                region = botocore.session.get_session().get_config_variable('region')
+            else:
+                module.fail_json("Boto3 is required for this module. Please install boto3 and try again")
 
     if not security_token:
         if 'AWS_SECURITY_TOKEN' in os.environ:
@@ -156,8 +162,7 @@ def get_aws_connection_info(module, boto3=False):
         boto_params = dict(aws_access_key_id=access_key,
                            aws_secret_access_key=secret_key,
                            aws_session_token=security_token)
-        if validate_certs:
-            boto_params['verify'] = validate_certs
+        boto_params['verify'] = validate_certs
 
         if profile_name:
             boto_params['profile_name'] = profile_name
@@ -174,7 +179,7 @@ def get_aws_connection_info(module, boto3=False):
                 module.fail_json("boto does not support profile_name before 2.24")
             boto_params['profile_name'] = profile_name
 
-        if validate_certs and HAS_LOOSE_VERSION and LooseVersion(boto.Version) >= LooseVersion("2.6.0"):
+        if HAS_LOOSE_VERSION and LooseVersion(boto.Version) >= LooseVersion("2.6.0"):
             boto_params['validate_certs'] = validate_certs
 
     for param, value in boto_params.items():
@@ -233,3 +238,27 @@ def ec2_connect(module):
         module.fail_json(msg="Either region or ec2_url must be specified")
 
     return ec2
+
+def paging(pause=0):
+    """ Adds paging to boto retrieval functions that support 'marker' """
+    def wrapper(f):
+        def page(*args, **kwargs):
+            results = []
+            marker = None
+            while True:
+                try:
+                    new = f(*args, marker=marker, **kwargs)
+                    marker = new.next_marker
+                    results.extend(new)
+                    if not marker:
+                        break
+                    elif pause:
+                        sleep(pause)
+                except TypeError:
+                    # Older version of boto do not allow for marker param, just run normally
+                    results = f(*args, **kwargs)
+                    break
+            return results
+        return page
+    return wrapper
+

lib/ansible/module_utils/eos.py

@@ -0,0 +1,259 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
+
+NET_COMMON_ARGS = dict(
+    host=dict(required=True),
+    port=dict(type='int'),
+    username=dict(required=True),
+    password=dict(no_log=True),
+    authorize=dict(default=False, type='bool'),
+    auth_pass=dict(no_log=True),
+    transport=dict(default='cli', choices=['cli', 'eapi']),
+    use_ssl=dict(default=True, type='bool'),
+    provider=dict(type='dict')
+)
+
+CLI_PROMPTS_RE = [
+    re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"),
+    re.compile(r"\[\w+\@[\w\-\.]+(?: [^\]])\] ?[>#\$] ?$")
+]
+
+CLI_ERRORS_RE = [
+    re.compile(r"% ?Error"),
+    re.compile(r"^% \w+", re.M),
+    re.compile(r"% ?Bad secret"),
+    re.compile(r"invalid input", re.I),
+    re.compile(r"(?:incomplete|ambiguous) command", re.I),
+    re.compile(r"connection timed out", re.I),
+    re.compile(r"[^\r\n]+ not found", re.I),
+    re.compile(r"'[^']' +returned error code: ?\d+"),
+    re.compile(r"[^\r\n]\/bin\/(?:ba)?sh")
+]
+
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+
+class Eapi(object):
+
+    def __init__(self, module):
+        self.module = module
+
+        # sets the module_utils/urls.py req parameters
+        self.module.params['url_username'] = module.params['username']
+        self.module.params['url_password'] = module.params['password']
+
+        self.url = None
+        self.enable = None
+
+    def _get_body(self, commands, encoding, reqid=None):
+        """Create a valid eAPI JSON-RPC request message
+        """
+        params = dict(version=1, cmds=commands, format=encoding)
+        return dict(jsonrpc='2.0', id=reqid, method='runCmds', params=params)
+
+    def connect(self):
+        host = self.module.params['host']
+        port = self.module.params['port']
+
+        if self.module.params['use_ssl']:
+            proto = 'https'
+            if not port:
+                port = 443
+        else:
+            proto = 'http'
+            if not port:
+                port = 80
+
+        self.url = '%s://%s:%s/command-api' % (proto, host, port)
+
+    def authorize(self):
+        if self.module.params['auth_pass']:
+            passwd = self.module.params['auth_pass']
+            self.enable = dict(cmd='enable', input=passwd)
+        else:
+            self.enable = 'enable'
+
+    def send(self, commands, encoding='json'):
+        """Send commands to the device.
+        """
+        clist = to_list(commands)
+
+        if self.enable is not None:
+            clist.insert(0, self.enable)
+
+        data = self._get_body(clist, encoding)
+        data = self.module.jsonify(data)
+
+        headers = {'Content-Type': 'application/json-rpc'}
+
+        response, headers = fetch_url(self.module, self.url, data=data,
+                headers=headers, method='POST')
+
+        if headers['status'] != 200:
+            self.module.fail_json(**headers)
+
+        response = self.module.from_json(response.read())
+        if 'error' in response:
+            err = response['error']
+            self.module.fail_json(msg='json-rpc error', **err)
+
+        if self.enable:
+            response['result'].pop(0)
+
+        return response['result']
+
+
+class Cli(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.shell = None
+
+    def connect(self, **kwargs):
+        host = self.module.params['host']
+        port = self.module.params['port'] or 22
+
+        username = self.module.params['username']
+        password = self.module.params['password']
+
+        try:
+            self.shell = Shell(CLI_PROMPTS_RE, CLI_ERRORS_RE)
+            self.shell.open(host, port=port, username=username, password=password)
+        except Exception, exc:
+            msg = 'failed to connecto to %s:%s - %s' % (host, port, str(exc))
+            self.module.fail_json(msg=msg)
+
+    def authorize(self):
+        passwd = self.module.params['auth_pass']
+        self.send(Command('enable', prompt=NET_PASSWD_RE, response=passwd))
+
+    def send(self, commands):
+        return self.shell.send(commands)
+
+
+class NetworkModule(AnsibleModule):
+
+    def __init__(self, *args, **kwargs):
+        super(NetworkModule, self).__init__(*args, **kwargs)
+        self.connection = None
+        self._config = None
+        self._connected = False
+
+    @property
+    def connected(self):
+        return self._connected
+
+    @property
+    def config(self):
+        if not self._config:
+            self._config = self.get_config()
+        return self._config
+
+    def _load_params(self):
+        params = super(NetworkModule, self)._load_params()
+        provider = params.get('provider') or dict()
+        for key, value in provider.items():
+            if key in NET_COMMON_ARGS.keys():
+                if not params.get(key) and value is not None:
+                    params[key] = value
+        return params
+
+    def connect(self):
+        if self.params['transport'] == 'eapi':
+            self.connection = Eapi(self)
+        else:
+            self.connection = Cli(self)
+
+        try:
+            self.connection.connect()
+            self.connection.send('terminal length 0')
+
+            if self.params['authorize']:
+                self.connection.authorize()
+
+        except Exception, exc:
+            self.fail_json(msg=exc.message)
+
+        self._connected = True
+
+    def configure(self, commands):
+        commands = to_list(commands)
+        commands.insert(0, 'configure terminal')
+        responses = self.execute(commands)
+        responses.pop(0)
+        return responses
+
+    def config_replace(self, commands):
+        if self.params['transport'] == 'cli':
+            self.fail_json(msg='config replace only supported over eapi')
+
+        cmd = 'configure replace terminal:'
+        commands = '\n'.join(to_list(commands))
+        command = dict(cmd=cmd, input=commands)
+        self.execute(command)
+
+    def execute(self, commands, **kwargs):
+        try:
+            if not self.connected:
+                self.connect()
+            return self.connection.send(commands, **kwargs)
+        except Exception, exc:
+            self.fail_json(msg=exc.message, commands=commands)
+
+    def disconnect(self):
+        self.connection.close()
+
+    def parse_config(self, cfg):
+        return parse(cfg, indent=3)
+
+    def get_config(self):
+        cmd = 'show running-config'
+        if self.params.get('include_defaults'):
+            cmd += ' all'
+        if self.params['transport'] == 'cli':
+            return self.execute(cmd)[0]
+        else:
+            resp = self.execute(cmd, encoding='text')
+            return resp[0]['output']
+
+
+def get_module(**kwargs):
+    """Return instance of NetworkModule
+    """
+    argument_spec = NET_COMMON_ARGS.copy()
+    if kwargs.get('argument_spec'):
+        argument_spec.update(kwargs['argument_spec'])
+    kwargs['argument_spec'] = argument_spec
+
+    module = NetworkModule(**kwargs)
+
+    # HAS_PARAMIKO is set by module_utils/shell.py
+    if module.params['transport'] == 'cli' and not HAS_PARAMIKO:
+        module.fail_json(msg='paramiko is required but does not appear to be installed')
+
+    return module
+

lib/ansible/module_utils/f5.py

@@ -51,19 +51,35 @@ def f5_argument_spec():
 def f5_parse_arguments(module):
     if not bigsuds_found:
         module.fail_json(msg="the python bigsuds module is required")
-    if not module.params['validate_certs']:
-        disable_ssl_cert_validation()
+
+    if module.params['validate_certs']:
+        import ssl
+        if not hasattr(ssl, 'SSLContext'):
+            module.fail_json(msg='bigsuds does not support verifying certificates with python < 2.7.9.  Either update python or set validate_certs=False on the task')
+
     return (module.params['server'],module.params['user'],module.params['password'],module.params['state'],module.params['partition'],module.params['validate_certs'])
 
-def bigip_api(bigip, user, password):
-    api = bigsuds.BIGIP(hostname=bigip, username=user, password=password)
-    return api
+def bigip_api(bigip, user, password, validate_certs):
+    try:
+        # bigsuds >= 1.0.3
+        api = bigsuds.BIGIP(hostname=bigip, username=user, password=password, verify=validate_certs)
+    except TypeError:
+        # bigsuds < 1.0.3, no verify param
+        if validate_certs:
+            # Note: verified we have SSLContext when we parsed params
+            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password)
+        else:
+            import ssl
+            if hasattr(ssl, 'SSLContext'):
+                # Really, you should never do this.  It disables certificate
+                # verification *globally*.  But since older bigip libraries
+                # don't give us a way to toggle verification we need to
+                # disable it at the global level.
+                # From https://www.python.org/dev/peps/pep-0476/#id29
+                ssl._create_default_https_context = ssl._create_unverified_context
+            api = bigsuds.BIGIP(hostname=bigip, username=user, password=password)
 
-def disable_ssl_cert_validation():
-    # You probably only want to do this for testing and never in production.
-    # From https://www.python.org/dev/peps/pep-0476/#id29
-    import ssl
-    ssl._create_default_https_context = ssl._create_unverified_context
+    return api
 
 # Fully Qualified name (with the partition)
 def fq_name(partition,name):

lib/ansible/module_utils/ios.py

@@ -16,180 +16,131 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
-"""
-Adds shared module support for connecting to and configuring Cisco
-IOS devices.  This shared module builds on module_utils/ssh.py and
-implements the Shell object.
 
-** Note: The order of the import statements does matter. **
+NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
 
-from ansible.module_utils.basic import *
-from ansible.module_utils.ssh import *
-from ansible.module_utils.ios import *
-
-This module provides the following common argument spec for creating
-ios connections:
-
-    * enable_mode (bool) - Forces the shell connection into IOS enable mode
-
-    * enable_password (str) - Configures the IOS enable mode password to be
-        send to the device to authorize the session
-
-    * device (dict) - Accepts the set of configuration parameters as a
-        dict object
-
-Note: These shared arguments are in addition to the arguments provided by
-the module_utils/ssh.py shared module
-
-"""
-import socket
-
-IOS_PROMPTS_RE = [
-    re.compile(r'[\r\n]?[a-zA-Z]{1}[a-zA-Z0-9-]*[>|#](?:\s*)$'),
-    re.compile(r'[\r\n]?[a-zA-Z]{1}[a-zA-Z0-9-]*\(.+\)#$'),
-    re.compile(r'\x1b.*$')
-]
-
-IOS_ERRORS_RE = [
-    re.compile(r"% ?Error"),
-    re.compile(r"^% \w+", re.M),
-    re.compile(r"% ?Bad secret"),
-    re.compile(r"invalid input", re.I),
-    re.compile(r"(?:incomplete|ambiguous) command", re.I),
-    re.compile(r"connection timed out", re.I),
-    re.compile(r"[^\r\n]+ not found", re.I),
-    re.compile(r"'[^']' +returned error code: ?\d+"),
-]
-
-IOS_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
-
-IOS_COMMON_ARGS = dict(
-    host=dict(),
-    port=dict(type='int', default=22),
-    username=dict(),
-    password=dict(),
-    enable_mode=dict(default=False, type='bool'),
-    enable_password=dict(),
-    connect_timeout=dict(type='int', default=10),
-    device=dict()
+NET_COMMON_ARGS = dict(
+    host=dict(required=True),
+    port=dict(default=22, type='int'),
+    username=dict(required=True),
+    password=dict(no_log=True),
+    authorize=dict(default=False, type='bool'),
+    auth_pass=dict(no_log=True),
+    provider=dict()
 )
 
 
-def ios_module(**kwargs):
-    """Append the common args to the argument_spec
-    """
-    spec = kwargs.get('argument_spec') or dict()
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
 
-    argument_spec = url_argument_spec()
-    argument_spec.update(IOS_COMMON_ARGS)
+
+class Cli(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.shell = None
+
+    def connect(self, **kwargs):
+        host = self.module.params['host']
+        port = self.module.params['port'] or 22
+
+        username = self.module.params['username']
+        password = self.module.params['password']
+
+        self.shell = Shell(kickstart=False)
+
+        try:
+            self.shell.open(host, port=port, username=username, password=password)
+        except Exception, exc:
+            msg = 'failed to connecto to %s:%s - %s' % (host, port, str(exc))
+            self.module.fail_json(msg=msg)
+
+    def authorize(self):
+        passwd = self.module.params['auth_pass']
+        self.send(Command('enable', prompt=NET_PASSWD_RE, response=passwd))
+
+    def send(self, commands):
+        return self.shell.send(commands)
+
+
+class NetworkModule(AnsibleModule):
+
+    def __init__(self, *args, **kwargs):
+        super(NetworkModule, self).__init__(*args, **kwargs)
+        self.connection = None
+        self._config = None
+
+    @property
+    def config(self):
+        if not self._config:
+            self._config = self.get_config()
+        return self._config
+
+    def _load_params(self):
+        params = super(NetworkModule, self)._load_params()
+        provider = params.get('provider') or dict()
+        for key, value in provider.items():
+            if key in NET_COMMON_ARGS.keys():
+                params[key] = value
+        return params
+
+    def connect(self):
+        try:
+            self.connection = Cli(self)
+            self.connection.connect()
+            self.execute('terminal length 0')
+
+            if self.params['authorize']:
+                self.connection.authorize()
+
+        except Exception, exc:
+            self.fail_json(msg=exc.message)
+
+    def configure(self, commands):
+        commands = to_list(commands)
+        commands.insert(0, 'configure terminal')
+        responses = self.execute(commands)
+        responses.pop(0)
+        return responses
+
+    def execute(self, commands, **kwargs):
+        try:
+            return self.connection.send(commands, **kwargs)
+        except Exception, exc:
+            self.fail_json(msg=exc.message, commands=commands)
+
+    def disconnect(self):
+        self.connection.close()
+
+    def parse_config(self, cfg):
+        return parse(cfg, indent=1)
+
+    def get_config(self):
+        cmd = 'show running-config'
+        if self.params.get('include_defaults'):
+            cmd += ' all'
+        return self.execute(cmd)[0]
+
+
+def get_module(**kwargs):
+    """Return instance of NetworkModule
+    """
+    argument_spec = NET_COMMON_ARGS.copy()
     if kwargs.get('argument_spec'):
         argument_spec.update(kwargs['argument_spec'])
     kwargs['argument_spec'] = argument_spec
 
-    module = AnsibleModule(**kwargs)
+    module = NetworkModule(**kwargs)
 
-    device = module.params.get('device') or dict()
-    for key, value in device.iteritems():
-        if key in IOS_COMMON_ARGS:
-            module.params[key] = value
-
-    params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS))
-    for key, value in params.iteritems():
-        if key != 'device':
-            module.params[key] = value
+    # HAS_PARAMIKO is set by module_utils/shell.py
+    if not HAS_PARAMIKO:
+        module.fail_json(msg='paramiko is required but does not appear to be installed')
 
+    module.connect()
     return module
 
-def to_list(arg):
-    """Try to force the arg to a list object
-    """
-    if isinstance(arg, (list, tuple)):
-        return list(arg)
-    elif arg is not None:
-        return [arg]
-    else:
-        return []
-
-class IosShell(object):
-
-    def __init__(self):
-        self.connection = None
-
-    def connect(self, host, username, password, **kwargs):
-        port = kwargs.get('port') or 22
-        timeout = kwargs.get('timeout') or 10
-
-        self.connection = Shell()
-
-        self.connection.prompts.extend(IOS_PROMPTS_RE)
-        self.connection.errors.extend(IOS_ERRORS_RE)
-
-        self.connection.open(host, port=port, username=username,
-                             password=password, timeout=timeout)
-
-    def authorize(self, passwd=None):
-        command = Command('enable', prompt=IOS_PASSWD_RE, response=passwd)
-        self.send(command)
-
-    def configure(self, commands):
-        commands = to_list(commands)
-
-        commands.insert(0, 'configure terminal')
-        commands.append('end')
-
-        resp = self.send(commands)
-        resp.pop(0)
-        resp.pop()
-
-        return resp
-
-    def send(self, commands):
-        responses = list()
-        for cmd in to_list(commands):
-            response = self.connection.send(cmd)
-            responses.append(response)
-        return responses
-
-def ios_from_args(module):
-    """Extracts the set of argumetns to build a valid IOS connection
-    """
-    params = dict()
-    for arg, attrs in IOS_COMMON_ARGS.iteritems():
-        if module.params['device']:
-            params[arg] = module.params['device'].get(arg)
-        if arg not in params or module.params[arg]:
-            params[arg] = module.params[arg]
-        if params[arg] is None:
-            if attrs.get('required'):
-                module.fail_json(msg='argument %s is required' % arg)
-            params[arg] = attrs.get('default')
-    return params
-
-def ios_connection(module):
-    """Creates a connection to an IOS device based on the module arguments
-    """
-    host = module.params['host']
-    port = module.params['port']
-
-    username = module.params['username']
-    password = module.params['password']
-
-    timeout = module.params['connect_timeout']
-
-    try:
-        shell = IosShell()
-        shell.connect(host, port=port, username=username, password=password,
-                    timeout=timeout)
-    except paramiko.ssh_exception.AuthenticationException, exc:
-        module.fail_json(msg=exc.message)
-    except socket.error, exc:
-        module.fail_json(msg=exc.strerror, errno=exc.errno)
-
-    shell.send('terminal length 0')
-
-    if module.params['enable_mode']:
-        shell.authorize(module.params['enable_password'])
-
-    return shell
-
-

lib/ansible/module_utils/iosxr.py

@@ -0,0 +1,127 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
+
+NET_COMMON_ARGS = dict(
+    host=dict(required=True),
+    port=dict(default=22, type='int'),
+    username=dict(required=True),
+    password=dict(no_log=True),
+    provider=dict()
+)
+
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+class Cli(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.shell = None
+
+    def connect(self, **kwargs):
+        host = self.module.params['host']
+        port = self.module.params['port'] or 22
+
+        username = self.module.params['username']
+        password = self.module.params['password']
+
+        self.shell = Shell()
+
+        try:
+            self.shell.open(host, port=port, username=username, password=password)
+        except Exception, exc:
+            msg = 'failed to connecto to %s:%s - %s' % (host, port, str(exc))
+            self.module.fail_json(msg=msg)
+
+    def send(self, commands):
+        return self.shell.send(commands)
+
+class NetworkModule(AnsibleModule):
+
+    def __init__(self, *args, **kwargs):
+        super(NetworkModule, self).__init__(*args, **kwargs)
+        self.connection = None
+        self._config = None
+
+    @property
+    def config(self):
+        if not self._config:
+            self._config = self.get_config()
+        return self._config
+
+    def _load_params(self):
+        params = super(NetworkModule, self)._load_params()
+        provider = params.get('provider') or dict()
+        for key, value in provider.items():
+            if key in NET_COMMON_ARGS.keys():
+                params[key] = value
+        return params
+
+    def connect(self):
+        try:
+            self.connection = Cli(self)
+            self.connection.connect()
+            self.execute('terminal length 0')
+        except Exception, exc:
+            self.fail_json(msg=exc.message)
+
+    def configure(self, commands):
+        commands = to_list(commands)
+        commands.insert(0, 'configure terminal')
+        commands.append('commit')
+        responses = self.execute(commands)
+        responses.pop(0)
+        responses.pop()
+        return responses
+
+    def execute(self, commands, **kwargs):
+        return self.connection.send(commands)
+
+    def disconnect(self):
+        self.connection.close()
+
+    def parse_config(self, cfg):
+        return parse(cfg, indent=1)
+
+    def get_config(self):
+        return self.execute('show running-config')[0]
+
+def get_module(**kwargs):
+    """Return instance of NetworkModule
+    """
+    argument_spec = NET_COMMON_ARGS.copy()
+    if kwargs.get('argument_spec'):
+        argument_spec.update(kwargs['argument_spec'])
+    kwargs['argument_spec'] = argument_spec
+
+    module = NetworkModule(**kwargs)
+
+    if not HAS_PARAMIKO:
+        module.fail_json(msg='paramiko is required but does not appear to be installed')
+
+    module.connect()
+    return module
+

lib/ansible/module_utils/ismount.py

@@ -0,0 +1,90 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is based on
+# Lib/posixpath.py of cpython
+# It is licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation
+# ("PSF"), and the Individual or Organization ("Licensee") accessing and
+# otherwise using this software ("Python") in source or binary form and
+# its associated documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
+# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+# analyze, test, perform and/or display publicly, prepare derivative works,
+# distribute, and otherwise use Python alone or in any derivative version,
+# provided, however, that PSF's License Agreement and PSF's notice of copyright,
+# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved"
+# are retained in Python alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on
+# or incorporates Python or any part thereof, and wants to make
+# the derivative work available to others as provided herein, then
+# Licensee hereby agrees to include in any such work a brief summary of
+# the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS"
+# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+# INFRINGE ANY THIRD PARTY RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material
+# breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any
+# relationship of agency, partnership, or joint venture between PSF and
+# Licensee.  This License Agreement does not grant permission to use PSF
+# trademarks or trade name in a trademark sense to endorse or promote
+# products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee
+# agrees to be bound by the terms and conditions of this License
+# Agreement.
+
+import os
+
+
+def ismount(path):
+    """Test whether a path is a mount point
+    clone of os.path.ismount (from cpython Lib/posixpath.py)
+    fixed to solve https://github.com/ansible/ansible-modules-core/issues/2186
+    and workaround non-fixed http://bugs.python.org/issue2466
+    this should be rewritten as soon as python issue 2466 is fixed
+    probably check for python version and use os.path.ismount if fixed
+
+    to remove replace in this file ismount( -> os.path.ismount( and remove this
+    function"""
+
+    try:
+        s1 = os.lstat(path)
+    except OSError:
+        # the OSError should be handled with more care
+        # it could be a "permission denied" but path is still a mount
+        return False
+    else:
+        # A symlink can never be a mount point
+        if os.path.stat.S_ISLNK(s1.st_mode):
+            return False
+
+    parent = os.path.join(path, os.path.pardir)
+    parent = os.path.realpath(parent)
+
+    try:
+        s2 = os.lstat(parent)
+    except OSError:
+        # one should handle the returned OSError with more care to figure
+        # out whether this is still a mount
+        return False
+
+    if s1.st_dev != s2.st_dev:
+        return True     # path/.. on a different device as path
+    if s1.st_ino == s2.st_ino:
+        return True     # path/.. is the same i-node as path, i.e. path=='/'
+    return False

lib/ansible/module_utils/junos.py

@@ -0,0 +1,128 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+NET_COMMON_ARGS = dict(
+    host=dict(required=True),
+    port=dict(default=22, type='int'),
+    username=dict(required=True),
+    password=dict(no_log=True),
+    provider=dict()
+)
+
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+class Cli(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.shell = None
+
+    def connect(self, **kwargs):
+        host = self.module.params['host']
+        port = self.module.params['port'] or 22
+
+        username = self.module.params['username']
+        password = self.module.params['password']
+
+        self.shell = Shell()
+
+        try:
+            self.shell.open(host, port=port, username=username, password=password)
+        except Exception, exc:
+            msg = 'failed to connecto to %s:%s - %s' % (host, port, str(exc))
+            self.module.fail_json(msg=msg)
+
+    def send(self, commands):
+        return self.shell.send(commands)
+
+
+class NetworkModule(AnsibleModule):
+
+    def __init__(self, *args, **kwargs):
+        super(NetworkModule, self).__init__(*args, **kwargs)
+        self.connection = None
+        self._config = None
+
+    @property
+    def config(self):
+        if not self._config:
+            self._config = self.get_config()
+        return self._config
+
+    def _load_params(self):
+        params = super(NetworkModule, self)._load_params()
+        provider = params.get('provider') or dict()
+        for key, value in provider.items():
+            if key in NET_COMMON_ARGS.keys():
+                params[key] = value
+        return params
+
+    def connect(self):
+        self.connection = Cli(self)
+        self.connection.connect()
+        if self.connection.shell._matched_prompt.strip().endswith('%'):
+            self.execute('cli')
+        self.execute('set cli screen-length 0')
+
+    def configure(self, commands):
+        commands = to_list(commands)
+        commands.insert(0, 'configure')
+        commands.append('commit and-quit')
+        responses = self.execute(commands)
+        responses.pop(0)
+        responses.pop()
+        return responses
+
+    def execute(self, commands, **kwargs):
+        return self.connection.send(commands)
+
+    def disconnect(self):
+        self.connection.close()
+
+    def parse_config(self, cfg):
+        return parse(cfg, indent=4)
+
+    def get_config(self):
+        cmd = 'show configuration'
+        return self.execute(cmd)[0]
+
+def get_module(**kwargs):
+    """Return instance of NetworkModule
+    """
+    argument_spec = NET_COMMON_ARGS.copy()
+    if kwargs.get('argument_spec'):
+        argument_spec.update(kwargs['argument_spec'])
+    kwargs['argument_spec'] = argument_spec
+    kwargs['check_invalid_arguments'] = False
+
+    module = NetworkModule(**kwargs)
+
+    # HAS_PARAMIKO is set by module_utils/shell.py
+    if not HAS_PARAMIKO:
+        module.fail_json(msg='paramiko is required but does not appear to be installed')
+
+    module.connect()
+    return module
+

lib/ansible/module_utils/known_hosts.py

@@ -28,7 +28,11 @@
 
 import os
 import hmac
-import urlparse
+
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
 
 try:
     from hashlib import sha1
@@ -74,12 +78,12 @@ def get_fqdn(repo_url):
     if "@" in repo_url and "://" not in repo_url:
         # most likely an user@host:path or user@host/path type URL
         repo_url = repo_url.split("@", 1)[1]
-        if ":" in repo_url:
-            repo_url = repo_url.split(":")[0]
-            result = repo_url
+        if repo_url.startswith('['):
+            result = repo_url.split(']', 1)[0] + ']'
+        elif ":" in repo_url:
+            result = repo_url.split(":")[0]
         elif "/" in repo_url:
-            repo_url = repo_url.split("/")[0]
-            result = repo_url
+            result = repo_url.split("/")[0]
     elif "://" in repo_url:
         # this should be something we can parse with urlparse
         parts = urlparse.urlparse(repo_url)
@@ -87,11 +91,13 @@ def get_fqdn(repo_url):
         # ensure we actually have a parts[1] before continuing.
         if parts[1] != '':
             result = parts[1]
-            if ":" in result:
-                result = result.split(":")[0]
             if "@" in result:
                 result = result.split("@", 1)[1]
 
+            if result[0].startswith('['):
+                result = result.split(']', 1)[0] + ']'
+            elif ":" in result:
+                result = result.split(":")[0]
     return result
 
 def check_hostkey(module, fqdn):
@@ -113,6 +119,7 @@ def not_in_host_file(self, host):
     host_file_list.append(user_host_file)
     host_file_list.append("/etc/ssh/ssh_known_hosts")
     host_file_list.append("/etc/ssh/ssh_known_hosts2")
+    host_file_list.append("/etc/openssh/ssh_known_hosts")
 
     hfiles_not_found = 0
     for hf in host_file_list:
@@ -169,7 +176,7 @@ def add_host_key(module, fqdn, key_type="rsa", create_dir=False):
     if not os.path.exists(user_ssh_dir):
         if create_dir:
             try:
-                os.makedirs(user_ssh_dir, 0700)
+                os.makedirs(user_ssh_dir, int('700', 8))
             except:
                 module.fail_json(msg="failed to create host key directory: %s" % user_ssh_dir)
         else:

lib/ansible/module_utils/mysql.py

@@ -0,0 +1,69 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright (c), Jonathan Mainguy <jon@soh.re>, 2015
+# Most of this was originally added by Sven Schliesing @muffl0n in the mysql_user.py module
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None, connect_timeout=30):
+    config = {
+        'host': module.params['login_host']
+    }
+
+    if ssl_ca is not None or ssl_key is not None or ssl_cert is not None:
+        config['ssl'] = {}
+
+    if module.params['login_unix_socket']:
+        config['unix_socket'] = module.params['login_unix_socket']
+    else:
+        config['port'] = module.params['login_port']
+
+    if os.path.exists(config_file):
+        config['read_default_file'] = config_file
+
+    # If login_user or login_password are given, they should override the
+    # config file
+    if login_user is not None:
+        config['user'] = login_user
+    if login_password is not None:
+        config['passwd'] = login_password
+    if ssl_cert is not None:
+        config['ssl']['cert'] = ssl_cert
+    if ssl_key is not None:
+        config['ssl']['key'] = ssl_key
+    if ssl_ca is not None:
+        config['ssl']['ca'] = ssl_ca
+    if db is not None:
+        config['db'] = db
+    if connect_timeout is not None:
+        config['connect_timeout'] = connect_timeout
+
+    db_connection = MySQLdb.connect(**config)
+    if cursor_class is not None:
+        return db_connection.cursor(cursorclass=MySQLdb.cursors.DictCursor)
+    else:
+        return db_connection.cursor()

lib/ansible/module_utils/netcfg.py

@@ -0,0 +1,176 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+import collections
+
+class ConfigLine(object):
+
+    def __init__(self, text):
+        self.text = text
+        self.children = list()
+        self.parents = list()
+        self.raw = None
+
+    def __str__(self):
+        return self.raw
+
+    def __eq__(self, other):
+        if self.text == other.text:
+            return self.parents == other.parents
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+def parse(lines, indent):
+    toplevel = re.compile(r'\S')
+    childline = re.compile(r'^\s*(.+)$')
+    repl = r'([{|}|;])'
+
+    ancestors = list()
+    config = list()
+
+    for line in str(lines).split('\n'):
+        text = str(re.sub(repl, '', line)).strip()
+
+        cfg = ConfigLine(text)
+        cfg.raw = line
+
+        if not text or text[0] in ['!', '#']:
+            continue
+
+        # handle top level commands
+        if toplevel.match(line):
+            ancestors = [cfg]
+
+        # handle sub level commands
+        else:
+            match = childline.match(line)
+            line_indent = match.start(1)
+            level = int(line_indent / indent)
+            parent_level = level - 1
+
+            cfg.parents = ancestors[:level]
+
+            if level > len(ancestors):
+                config.append(cfg)
+                continue
+
+            for i in range(level, len(ancestors)):
+                ancestors.pop()
+
+            ancestors.append(cfg)
+            ancestors[parent_level].children.append(cfg)
+
+        config.append(cfg)
+
+    return config
+
+
+class Conditional(object):
+    """Used in command modules to evaluate waitfor conditions
+    """
+
+    OPERATORS = {
+        'eq': ['eq', '=='],
+        'neq': ['neq', 'ne', '!='],
+        'gt': ['gt', '>'],
+        'ge': ['ge', '>='],
+        'lt': ['lt', '<'],
+        'le': ['le', '<='],
+        'contains': ['contains']
+    }
+
+    def __init__(self, conditional):
+        self.raw = conditional
+
+        key, op, val = shlex.split(conditional)
+        self.key = key
+        self.func = self.func(op)
+        self.value = self._cast_value(val)
+
+    def __call__(self, data):
+        try:
+            value = self.get_value(dict(result=data))
+            return self.func(value)
+        except Exception:
+            raise ValueError(self.key)
+
+    def _cast_value(self, value):
+        if value in BOOLEANS_TRUE:
+            return True
+        elif value in BOOLEANS_FALSE:
+            return False
+        elif re.match(r'^\d+\.d+$', value):
+            return float(value)
+        elif re.match(r'^\d+$', value):
+            return int(value)
+        else:
+            return unicode(value)
+
+    def func(self, oper):
+        for func, operators in self.OPERATORS.items():
+            if oper in operators:
+                return getattr(self, func)
+        raise AttributeError('unknown operator: %s' % oper)
+
+    def get_value(self, result):
+        parts = re.split(r'\.(?=[^\]]*(?:\[|$))', self.key)
+        for part in parts:
+            match = re.findall(r'\[(\S+?)\]', part)
+            if match:
+                key = part[:part.find('[')]
+                result = result[key]
+                for m in match:
+                    try:
+                        m = int(m)
+                    except ValueError:
+                        m = str(m)
+                    result = result[m]
+            else:
+                result = result.get(part)
+        return result
+
+    def number(self, value):
+        if '.' in str(value):
+            return float(value)
+        else:
+            return int(value)
+
+    def eq(self, value):
+        return value == self.value
+
+    def neq(self, value):
+        return value != self.value
+
+    def gt(self, value):
+        return self.number(value) > self.value
+
+    def ge(self, value):
+        return self.number(value) >= self.value
+
+    def lt(self, value):
+        return self.number(value) < self.value
+
+    def le(self, value):
+        return self.number(value) <= self.value
+
+    def contains(self, value):
+        return self.value in value

lib/ansible/module_utils/nxapi.py

@@ -1,130 +0,0 @@
-#
-# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
-"""
-This module adds support for Cisco NXAPI to Ansible shared
-module_utils.  It builds on module_utils/urls.py to provide
-NXAPI support over HTTP/S which is required for proper operation.
-
-In order to use this module, include it as part of a custom
-module as shown below.
-
-** Note: The order of the import statements does matter. **
-
-from ansible.module_utils.basic import *
-from ansible.module_utils.urls import *
-from ansible.module_utils.nxapi import *
-
-The nxapi module provides the following common argument spec:
-
-    * host (str) - [Required] The IPv4 address or FQDN of the network device
-
-    * port (str) - Overrides the default port to use for the HTTP/S
-        connection.  The default values are 80 for HTTP and
-        443 for HTTPS
-
-    * url_username (str) - [Required] The username to use to authenticate
-        the HTTP/S connection.  Aliases: username
-
-    * url_password (str) - [Required] The password to use to authenticate
-        the HTTP/S connection.  Aliases: password
-
-    * use_ssl (bool) - Specifies whether or not to use an encrypted (HTTPS)
-        connection or not.  The default value is False.
-
-    * command_type (str) - The type of command to send to the remote
-        device.  Valid values in `cli_show`, `cli_show_ascii`, 'cli_conf`
-        and `bash`.  The default value is `cli_show_ascii`
-
-In order to communicate with Cisco NXOS devices, the NXAPI feature
-must be enabled and configured on the device.
-
-"""
-
-NXAPI_COMMAND_TYPES = ['cli_show', 'cli_show_ascii', 'cli_conf', 'bash']
-
-def nxapi_argument_spec(spec=None):
-    """Creates an argument spec for working with NXAPI
-    """
-    arg_spec = url_argument_spec()
-    arg_spec.update(dict(
-        host=dict(required=True),
-        port=dict(),
-        url_username=dict(required=True, aliases=['username']),
-        url_password=dict(required=True, aliases=['password']),
-        use_ssl=dict(default=False, type='bool'),
-        command_type=dict(default='cli_show_ascii', choices=NXAPI_COMMAND_TYPES)
-    ))
-    if spec:
-        arg_spec.update(spec)
-    return arg_spec
-
-def nxapi_url(module):
-    """Constructs a valid NXAPI url
-    """
-    if module.params['use_ssl']:
-        proto = 'https'
-    else:
-        proto = 'http'
-    host = module.params['host']
-    url = '{}://{}'.format(proto, host)
-    port = module.params['port']
-    if module.params['port']:
-        url = '{}:{}'.format(url, module.params['port'])
-    url = '{}/ins'.format(url)
-    return url
-
-def nxapi_body(commands, command_type, **kwargs):
-    """Encodes a NXAPI JSON request message
-    """
-    if isinstance(commands, (list, set, tuple)):
-        commands = ' ;'.join(commands)
-
-    msg = {
-        'version': kwargs.get('version') or '1.2',
-        'type': command_type,
-        'chunk': kwargs.get('chunk') or '0',
-        'sid': kwargs.get('sid'),
-        'input': commands,
-        'output_format': 'json'
-    }
-
-    return dict(ins_api=msg)
-
-def nxapi_command(module, commands, command_type=None, **kwargs):
-    """Sends the list of commands to the device over NXAPI
-    """
-    url = nxapi_url(module)
-
-    command_type = command_type or module.params['command_type']
-
-    data = nxapi_body(commands, command_type)
-    data = module.jsonify(data)
-
-    headers = {'Content-Type': 'text/json'}
-
-    response, headers = fetch_url(module, url, data=data, headers=headers,
-                                  method='POST')
-
-    status = kwargs.get('status') or 200
-    if headers['status'] != status:
-        module.fail_json(**headers)
-
-    response = module.from_json(response.read())
-    return response, headers
-

lib/ansible/module_utils/nxos.py

@@ -0,0 +1,244 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
+
+NET_COMMON_ARGS = dict(
+    host=dict(required=True),
+    port=dict(type='int'),
+    username=dict(required=True),
+    password=dict(no_log=True),
+    transport=dict(default='cli', choices=['cli', 'nxapi']),
+    use_ssl=dict(default=False, type='bool'),
+    validate_certs=dict(default=True, type='bool'),
+    provider=dict(type='dict')
+)
+
+NXAPI_COMMAND_TYPES = ['cli_show', 'cli_show_ascii', 'cli_conf', 'bash']
+
+NXAPI_ENCODINGS = ['json', 'xml']
+
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+class Nxapi(object):
+
+    def __init__(self, module):
+        self.module = module
+
+        # sets the module_utils/urls.py req parameters
+        self.module.params['url_username'] = module.params['username']
+        self.module.params['url_password'] = module.params['password']
+
+        self.url = None
+        self._nxapi_auth = None
+
+    def _get_body(self, commands, command_type, encoding, version='1.2', chunk='0', sid=None):
+        """Encodes a NXAPI JSON request message
+        """
+        if isinstance(commands, (list, set, tuple)):
+            commands = ' ;'.join(commands)
+
+        if encoding not in NXAPI_ENCODINGS:
+            msg = 'invalid encoding, received %s, exceped one of %s' % \
+                    (encoding, ','.join(NXAPI_ENCODINGS))
+            self.module_fail_json(msg=msg)
+
+        msg = {
+            'version': version,
+            'type': command_type,
+            'chunk': chunk,
+            'sid': sid,
+            'input': commands,
+            'output_format': encoding
+        }
+        return dict(ins_api=msg)
+
+    def connect(self):
+        host = self.module.params['host']
+        port = self.module.params['port']
+
+        if self.module.params['use_ssl']:
+            proto = 'https'
+            if not port:
+                port = 443
+        else:
+            proto = 'http'
+            if not port:
+                port = 80
+
+        self.url = '%s://%s:%s/ins' % (proto, host, port)
+
+    def send(self, commands, command_type='cli_show_ascii', encoding='json'):
+        """Send commands to the device.
+        """
+        clist = to_list(commands)
+
+        if command_type not in NXAPI_COMMAND_TYPES:
+            msg = 'invalid command_type, received %s, exceped one of %s' % \
+                    (command_type, ','.join(NXAPI_COMMAND_TYPES))
+            self.module_fail_json(msg=msg)
+
+        debug = dict()
+
+        data = self._get_body(clist, command_type, encoding)
+        data = self.module.jsonify(data)
+
+        headers = {'Content-Type': 'application/json'}
+        if self._nxapi_auth:
+            headers['Cookie'] = self._nxapi_auth
+
+        response, headers = fetch_url(self.module, self.url, data=data,
+                headers=headers, method='POST')
+
+        self._nxapi_auth = headers.get('set-cookie')
+
+        if headers['status'] != 200:
+            self.module.fail_json(**headers)
+
+        response = self.module.from_json(response.read())
+        result = list()
+
+        output = response['ins_api']['outputs']['output']
+        for item in to_list(output):
+            if item['code'] != '200':
+                self.module.fail_json(**item)
+            else:
+                result.append(item['body'])
+
+        return result
+
+
+class Cli(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.shell = None
+
+    def connect(self, **kwargs):
+        host = self.module.params['host']
+        port = self.module.params['port'] or 22
+
+        username = self.module.params['username']
+        password = self.module.params['password']
+
+        try:
+            self.shell = Shell()
+            self.shell.open(host, port=port, username=username, password=password)
+        except Exception, exc:
+            msg = 'failed to connect to %s:%s - %s' % (host, port, str(exc))
+            self.module.fail_json(msg=msg)
+
+    def send(self, commands, encoding='text'):
+        return self.shell.send(commands)
+
+
+class NetworkModule(AnsibleModule):
+
+    def __init__(self, *args, **kwargs):
+        super(NetworkModule, self).__init__(*args, **kwargs)
+        self.connection = None
+        self._config = None
+        self._connected = False
+
+    @property
+    def connected(self):
+        return self._connected
+
+    @property
+    def config(self):
+        if not self._config:
+            self._config = self.get_config()
+        return self._config
+
+    def _load_params(self):
+        params = super(NetworkModule, self)._load_params()
+        provider = params.get('provider') or dict()
+        for key, value in provider.items():
+            if key in NET_COMMON_ARGS.keys():
+                if not params.get(key) and value is not None:
+                    params[key] = value
+        return params
+
+    def connect(self):
+        if self.params['transport'] == 'nxapi':
+            self.connection = Nxapi(self)
+        else:
+            self.connection = Cli(self)
+
+        self.connection.connect()
+
+        if self.params['transport'] == 'cli':
+            self.connection.send('terminal length 0')
+
+        self._connected = True
+
+
+    def configure_cli(self, commands):
+        commands = to_list(commands)
+        commands.insert(0, 'configure')
+        responses = self.execute(commands)
+        responses.pop(0)
+        return responses
+
+    def configure(self, commands):
+        commands = to_list(commands)
+        if self.params['transport'] == 'cli':
+            return self.configure_cli(commands)
+        else:
+            return self.execute(commands, command_type='cli_conf')
+
+    def execute(self, commands, **kwargs):
+        if not self.connected:
+            self.connect()
+        return self.connection.send(commands, **kwargs)
+
+    def disconnect(self):
+        self.connection.close()
+
+    def parse_config(self, cfg):
+        return parse(cfg, indent=2)
+
+    def get_config(self):
+        cmd = 'show running-config'
+        if self.params.get('include_defaults'):
+            cmd += ' all'
+        response = self.execute(cmd)
+        return response[0]
+
+
+def get_module(**kwargs):
+    """Return instance of NetworkModule
+    """
+    argument_spec = NET_COMMON_ARGS.copy()
+    if kwargs.get('argument_spec'):
+        argument_spec.update(kwargs['argument_spec'])
+    kwargs['argument_spec'] = argument_spec
+
+    module = NetworkModule(**kwargs)
+
+    # HAS_PARAMIKO is set by module_utils/shell.py
+    if module.params['transport'] == 'cli' and not HAS_PARAMIKO:
+        module.fail_json(msg='paramiko is required but does not appear to be installed')
+
+    return module

lib/ansible/module_utils/openstack.py

@@ -74,10 +74,10 @@ def openstack_full_argument_spec(**kwargs):
     spec = dict(
         cloud=dict(default=None),
         auth_type=dict(default=None),
-        auth=dict(default=None, no_log=True),
+        auth=dict(default=None, type='dict', no_log=True),
         region_name=dict(default=None),
         availability_zone=dict(default=None),
-        verify=dict(default=True, aliases=['validate_certs']),
+        verify=dict(default=True, type='bool', aliases=['validate_certs']),
         cacert=dict(default=None),
         cert=dict(default=None),
         key=dict(default=None, no_log=True),

lib/ansible/module_utils/openswitch.py

@@ -0,0 +1,250 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+import time
+import json
+
+try:
+    from runconfig import runconfig
+    from opsrest.settings import settings
+    from opsrest.manager import OvsdbConnectionManager
+    from opslib import restparser
+    HAS_OPS = True
+except ImportError:
+    HAS_OPS = False
+
+NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
+
+NET_COMMON_ARGS = dict(
+    host=dict(),
+    port=dict(type='int'),
+    username=dict(),
+    password=dict(no_log=True),
+    use_ssl=dict(default=True, type='bool'),
+    transport=dict(default='ssh', choices=['ssh', 'cli', 'rest']),
+    provider=dict()
+)
+
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+def get_runconfig():
+    manager = OvsdbConnectionManager(settings.get('ovs_remote'),
+                                     settings.get('ovs_schema'))
+    manager.start()
+
+    timeout = 10
+    interval = 0
+    init_seq_no = manager.idl.change_seqno
+
+    while (init_seq_no == manager.idl.change_seqno):
+        if interval > timeout:
+            raise TypeError('timeout')
+        manager.idl.run()
+        interval += 1
+        time.sleep(1)
+
+    schema = restparser.parseSchema(settings.get('ext_schema'))
+    return runconfig.RunConfigUtil(manager.idl, schema)
+
+class Response(object):
+
+    def __init__(self, resp, hdrs):
+        self.body = None
+        self.headers = hdrs
+
+        if resp:
+            self.body = resp.read()
+
+    @property
+    def json(self):
+        if not self.body:
+            return None
+        try:
+            return json.loads(self.body)
+        except ValueError:
+            return None
+
+class Rest(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.baseurl = None
+
+    def connect(self):
+        host = self.module.params['host']
+        port = self.module.params['port']
+
+        if self.module.params['use_ssl']:
+            proto = 'https'
+            if not port:
+                port = 18091
+        else:
+            proto = 'http'
+            if not port:
+                port = 8091
+
+        self.baseurl = '%s://%s:%s/rest/v1' % (proto, host, port)
+
+    def _url_builder(self, path):
+        if path[0] == '/':
+            path = path[1:]
+        return '%s/%s' % (self.baseurl, path)
+
+    def send(self, method, path, data=None, headers=None):
+        url = self._url_builder(path)
+        data = self.module.jsonify(data)
+
+        if headers is None:
+            headers = dict()
+        headers.update({'Content-Type': 'application/json'})
+
+        resp, hdrs = fetch_url(self.module, url, data=data, headers=headers,
+                method=method)
+
+        return Response(resp, hdrs)
+
+    def get(self, path, data=None, headers=None):
+        return self.send('GET', path, data, headers)
+
+    def put(self, path, data=None, headers=None):
+        return self.send('PUT', path, data, headers)
+
+    def post(self, path, data=None, headers=None):
+        return self.send('POST', path, data, headers)
+
+    def delete(self, path, data=None, headers=None):
+        return self.send('DELETE', path, data, headers)
+
+class Cli(object):
+
+    def __init__(self, module):
+        self.module = module
+        self.shell = None
+
+    def connect(self, **kwargs):
+        host = self.module.params['host']
+        port = self.module.params['port'] or 22
+
+        username = self.module.params['username']
+        password = self.module.params['password']
+
+        self.shell = Shell()
+        self.shell.open(host, port=port, username=username, password=password)
+
+    def send(self, commands, encoding='text'):
+        return self.shell.send(commands)
+
+class NetworkModule(AnsibleModule):
+
+    def __init__(self, *args, **kwargs):
+        super(NetworkModule, self).__init__(*args, **kwargs)
+        self.connection = None
+        self._config = None
+        self._runconfig = None
+
+    @property
+    def config(self):
+        if not self._config:
+            self._config = self.get_config()
+        return self._config
+
+    def _load_params(self):
+        params = super(NetworkModule, self)._load_params()
+        provider = params.get('provider') or dict()
+        for key, value in provider.items():
+            if key in NET_COMMON_ARGS.keys():
+                params[key] = value
+        return params
+
+    def connect(self):
+        if self.params['transport'] == 'rest':
+            self.connection = Rest(self)
+        elif self.params['transport'] == 'cli':
+            self.connection = Cli(self)
+
+        self.connection.connect()
+
+    def configure(self, config):
+        if self.params['transport'] == 'cli':
+            commands = to_list(config)
+            commands.insert(0, 'configure terminal')
+            responses = self.execute(commands)
+            responses.pop(0)
+            return responses
+        elif self.params['transport'] == 'rest':
+            path = '/system/full-configuration'
+            return self.connection.put(path, data=config)
+        else:
+            if not self._runconfig:
+                self._runconfig = get_runconfig()
+            self._runconfig.write_config_to_db(config)
+
+    def execute(self, commands, **kwargs):
+        try:
+            return self.connection.send(commands, **kwargs)
+        except Exception, exc:
+            self.fail_json(msg=exc.message, commands=commands)
+
+    def disconnect(self):
+        self.connection.close()
+
+    def parse_config(self, cfg):
+        return parse(cfg, indent=4)
+
+    def get_config(self):
+        if self.params['transport'] == 'cli':
+            return self.execute('show running-config')[0]
+
+        elif self.params['transport'] == 'rest':
+            resp = self.connection.get('/system/full-configuration')
+            return resp.json
+
+        else:
+            if not self._runconfig:
+                self._runconfig = get_runconfig()
+            return self._runconfig.get_running_config()
+
+
+def get_module(**kwargs):
+    """Return instance of NetworkModule
+    """
+    argument_spec = NET_COMMON_ARGS.copy()
+    if kwargs.get('argument_spec'):
+        argument_spec.update(kwargs['argument_spec'])
+    kwargs['argument_spec'] = argument_spec
+
+    module = NetworkModule(**kwargs)
+
+    if not HAS_OPS and module.params['transport'] == 'ssh':
+        module.fail_json(msg='could not import ops library')
+
+    # HAS_PARAMIKO is set by module_utils/shell.py
+    if module.params['transport'] == 'cli' and not HAS_PARAMIKO:
+        module.fail_json(msg='paramiko is required but does not appear to be installed')
+
+    if module.params['transport'] in ['cli', 'rest']:
+        module.connect()
+
+    return module
+

lib/ansible/module_utils/powershell.ps1

@@ -211,7 +211,7 @@ Function Get-FileChecksum($path)
     If (Test-Path -PathType Leaf $path)
     {
         $sp = new-object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider;
-        $fp = [System.IO.File]::Open($path, [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
+        $fp = [System.IO.File]::Open($path, [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite);
         $hash = [System.BitConverter]::ToString($sp.ComputeHash($fp)).Replace("-", "").ToLower();
         $fp.Dispose();
     }

lib/ansible/module_utils/shell.py

@@ -0,0 +1,187 @@
+#
+# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+import re
+import socket
+
+# py2 vs py3; replace with six via ziploader
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+try:
+    import paramiko
+    HAS_PARAMIKO = True
+except ImportError:
+    HAS_PARAMIKO = False
+
+
+ANSI_RE = re.compile(r'(\x1b\[\?1h\x1b=)')
+
+CLI_PROMPTS_RE = [
+    re.compile(r'[\r\n]?[a-zA-Z]{1}[a-zA-Z0-9-]*[>|#|%](?:\s*)$'),
+    re.compile(r'[\r\n]?[a-zA-Z]{1}[a-zA-Z0-9-]*\(.+\)#(?:\s*)$')
+]
+
+CLI_ERRORS_RE = [
+    re.compile(r"% ?Error"),
+    re.compile(r"^% \w+", re.M),
+    re.compile(r"% ?Bad secret"),
+    re.compile(r"invalid input", re.I),
+    re.compile(r"(?:incomplete|ambiguous) command", re.I),
+    re.compile(r"connection timed out", re.I),
+    re.compile(r"[^\r\n]+ not found", re.I),
+    re.compile(r"'[^']' +returned error code: ?\d+"),
+    re.compile(r"syntax error"),
+    re.compile(r"unknown command")
+]
+
+def to_list(val):
+    if isinstance(val, (list, tuple)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+class ShellError(Exception):
+
+    def __init__(self, msg, command=None):
+        super(ShellError, self).__init__(msg)
+        self.message = msg
+        self.command = command
+
+class Command(object):
+
+    def __init__(self, command, prompt=None, response=None):
+        self.command = command
+        self.prompt = prompt
+        self.response = response
+
+    def __str__(self):
+        return self.command
+
+class Shell(object):
+
+    def __init__(self, prompts_re=None, errors_re=None, kickstart=True):
+        self.ssh = None
+        self.shell = None
+
+        self.kickstart = kickstart
+        self._matched_prompt = None
+
+        self.prompts = prompts_re or CLI_PROMPTS_RE
+        self.errors = errors_re or CLI_ERRORS_RE
+
+    def open(self, host, port=22, username=None, password=None,
+            timeout=10, key_filename=None, pkey=None, look_for_keys=None,
+            allow_agent=False):
+
+        self.ssh = paramiko.SSHClient()
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        # unless explicitly set, disable look for keys if a password is
+        # present. this changes the default search order paramiko implements
+        if not look_for_keys:
+            look_for_keys = password is None
+
+        self.ssh.connect(host, port=port, username=username, password=password,
+                    timeout=timeout, look_for_keys=look_for_keys, pkey=pkey,
+                    key_filename=key_filename, allow_agent=allow_agent)
+
+        self.shell = self.ssh.invoke_shell()
+        self.shell.settimeout(10)
+
+        if self.kickstart:
+            self.shell.sendall("\n")
+
+        self.receive()
+
+    def strip(self, data):
+        return ANSI_RE.sub('', data)
+
+    def receive(self, cmd=None):
+        recv = StringIO()
+
+        while True:
+            data = self.shell.recv(200)
+
+            recv.write(data)
+            recv.seek(recv.tell() - 200)
+
+            window = self.strip(recv.read())
+
+            if isinstance(cmd, Command):
+                self.handle_input(window, prompt=cmd.prompt,
+                                  response=cmd.response)
+
+            try:
+                if self.read(window):
+                    resp = self.strip(recv.getvalue())
+                    return self.sanitize(cmd, resp)
+            except ShellError, exc:
+                exc.command = cmd
+                raise
+
+    def send(self, commands):
+        responses = list()
+        try:
+            for command in to_list(commands):
+                cmd = '%s\r' % str(command)
+                self.shell.sendall(cmd)
+                responses.append(self.receive(command))
+        except socket.timeout, exc:
+            raise ShellError("timeout trying to send command", cmd)
+        return responses
+
+    def close(self):
+        self.shell.close()
+
+    def handle_input(self, resp, prompt, response):
+        if not prompt or not response:
+            return
+
+        prompt = to_list(prompt)
+        response = to_list(response)
+
+        for pr, ans in zip(prompt, response):
+            match = pr.search(resp)
+            if match:
+                cmd = '%s\r' % ans
+                self.shell.sendall(cmd)
+
+    def sanitize(self, cmd, resp):
+        cleaned = []
+        for line in resp.splitlines():
+            if line.startswith(str(cmd)) or self.read(line):
+                continue
+            cleaned.append(line)
+        return "\n".join(cleaned)
+
+    def read(self, response):
+        for regex in self.errors:
+            if regex.search(response):
+                raise ShellError('%s' % response)
+
+        for regex in self.prompts:
+            match = regex.search(response)
+            if match:
+                self._matched_prompt = match.group()
+                return True
+

lib/ansible/module_utils/ssh.py

@@ -1,189 +0,0 @@
-#
-# (c) 2015 Peter Sprygada, <psprygada@ansible.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
-"""
-Ansible shared module for building modules that require an interactive
-SSH Shell such as those for command line driven devices.  This module
-provides a native SSH transport using paramiko and builds a base Shell
-class for creating shell driven modules.
-
-In order to use this module, include it as part of a custom
-module as shown below and create and subclass Shell.
-
-** Note: The order of the import statements does matter. **
-
-from ansible.module_utils.basic import *
-from ansible.module_utils.ssh import *
-
-This module provides the following common argument spec for creating
-shell connections:
-
-    * host (str) - [Required]  The IPv4 address or FQDN of the device
-
-    * port (int) - Overrides the default SSH port.
-
-    * username (str) - [Required] The username to use to authenticate
-        the SSH session.
-
-    * password (str) - [Required] The password to use to authenticate
-        the SSH session
-
-    * connect_timeout (int) - Specifies the connection timeout in seconds
-
-"""
-import re
-import socket
-
-from StringIO import StringIO
-
-import paramiko
-
-def shell_argument_spec(spec=None):
-    """ Generates an argument spec for the Shell class
-    """
-    arg_spec = dict(
-        host=dict(required=True),
-        port=dict(default=22, type='int'),
-        username=dict(required=True),
-        password=dict(required=True),
-        connect_timeout=dict(default=10, type='int'),
-    )
-    if spec:
-        arg_spec.update(spec)
-    return arg_spec
-
-
-class ShellError(Exception):
-
-    def __init__(self, msg, command=None):
-        super(ShellError, self).__init__(msg)
-        self.message = msg
-        self.command = command
-
-
-class Command(object):
-
-    def __init__(self, command, prompt=None, response=None):
-        self.command = command
-        self.prompt = prompt
-        self.response = response
-
-    def __str__(self):
-        return self.command
-
-class Ssh(object):
-
-    def __init__(self):
-        self.client = None
-
-    def open(self, host, port=22, username=None, password=None, timeout=10):
-        ssh = paramiko.SSHClient()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        ssh.connect(host, port=port, username=username, password=password,
-                    timeout=timeout, allow_agent=False, look_for_keys=False)
-
-        self.client = ssh
-        return self.on_open()
-
-    def on_open(self):
-        pass
-
-    def close(self):
-        self.client.close()
-        return self.on_close()
-
-    def on_close(self):
-        pass
-
-
-class Shell(Ssh):
-
-    def __init__(self):
-        super(Shell, self).__init__()
-        self.shell = None
-
-        self.prompts = list()
-        self.errors = list()
-
-    def on_open(self):
-        self.shell = self.client.invoke_shell()
-        self.shell.settimeout(10)
-        self.receive()
-
-    def receive(self, cmd=None):
-        recv = StringIO()
-
-        while True:
-            recv.write(self.shell.recv(200))
-            recv.seek(recv.tell() - 200)
-
-            window = recv.read()
-
-            if isinstance(cmd, Command):
-                self.handle_input(window, prompt=cmd.prompt,
-                                  response=cmd.response)
-
-            try:
-                if self.read(window):
-                    resp = recv.getvalue()
-                    return self.sanitize(cmd, resp)
-            except ShellError, exc:
-                exc.command = cmd
-                raise
-
-    def send(self, command):
-        try:
-            cmd = '%s\r' % str(command)
-            self.shell.sendall(cmd)
-            return self.receive(command)
-        except socket.timeout, exc:
-            raise ShellError("timeout trying to send command", cmd)
-
-    def handle_input(self, resp, prompt, response):
-        if not prompt or not response:
-            return
-
-        prompt = to_list(prompt)
-        response = to_list(response)
-
-        for pr, ans in zip(prompt, response):
-            match = pr.search(resp)
-            if match:
-                cmd = '%s\r' % ans
-                self.shell.sendall(cmd)
-
-    def sanitize(self, cmd, resp):
-        cleaned = []
-        for line in resp.splitlines():
-            if line.startswith(str(cmd)) or self.read(line):
-                continue
-            cleaned.append(line)
-        return "\n".join(cleaned)
-
-    def read(self, response):
-        for regex in self.errors:
-            if regex.search(response):
-                raise ShellError('{}'.format(response))
-
-        for regex in self.prompts:
-            if regex.search(response):
-                return True
-
-
-

lib/ansible/module_utils/urls.py

@@ -310,36 +310,45 @@ class NoSSLError(SSLValidationError):
     """Needed to connect to an HTTPS url but no ssl library available to verify the certificate"""
     pass
 
+# Some environments (Google Compute Engine's CoreOS deploys) do not compile
+# against openssl and thus do not have any HTTPS support.
+CustomHTTPSConnection = CustomHTTPSHandler = None
+if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib2, 'HTTPSHandler'):
+    class CustomHTTPSConnection(httplib.HTTPSConnection):
+        def __init__(self, *args, **kwargs):
+            httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+            if HAS_SSLCONTEXT:
+                self.context = create_default_context()
+                if self.cert_file:
+                    self.context.load_cert_chain(self.cert_file, self.key_file)
 
-class CustomHTTPSConnection(httplib.HTTPSConnection):
-    def __init__(self, *args, **kwargs):
-        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
-        if HAS_SSLCONTEXT:
-            self.context = create_default_context()
-            if self.cert_file:
-                self.context.load_cert_chain(self.cert_file, self.key_file)
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
 
-    def connect(self):
-        "Connect to a host on a given (SSL) port."
+            if hasattr(self, 'source_address'):
+                sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address)
+            else:
+                sock = socket.create_connection((self.host, self.port), self.timeout)
 
-        if hasattr(self, 'source_address'):
-            sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address)
-        else:
-            sock = socket.create_connection((self.host, self.port), self.timeout)
-        if self._tunnel_host:
-            self.sock = sock
-            self._tunnel()
-        if HAS_SSLCONTEXT:
-            self.sock = self.context.wrap_socket(sock, server_hostname=self.host)
-        else:
-            self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL)
+            server_hostname = self.host
+            # Note: self._tunnel_host is not available on py < 2.6 but this code
+            # isn't used on py < 2.6 (lack of create_connection)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+                server_hostname = self._tunnel_host
 
-class CustomHTTPSHandler(urllib2.HTTPSHandler):
+            if HAS_SSLCONTEXT:
+                self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname)
+            else:
+                self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL)
 
-    def https_open(self, req):
-        return self.do_open(CustomHTTPSConnection, req)
+    class CustomHTTPSHandler(urllib2.HTTPSHandler):
 
-    https_request = urllib2.AbstractHTTPHandler.do_request_
+        def https_open(self, req):
+            return self.do_open(CustomHTTPSConnection, req)
+
+        https_request = urllib2.AbstractHTTPHandler.do_request_
 
 def generic_urlparse(parts):
     '''
@@ -373,7 +382,10 @@ def generic_urlparse(parts):
         # get the username, password, etc.
         try:
             netloc_re = re.compile(r'^((?:\w)+(?::(?:\w)+)?@)?([A-Za-z0-9.-]+)(:\d+)?$')
-            (auth, hostname, port) = netloc_re.match(parts[1])
+            match = netloc_re.match(parts[1])
+            auth = match.group(1)
+            hostname = match.group(2)
+            port = match.group(3)
             if port:
                 # the capture group for the port will include the ':',
                 # so remove it and convert the port to an integer
@@ -383,6 +395,8 @@ def generic_urlparse(parts):
                 # and then split it up based on the first ':' found
                 auth = auth[:-1]
                 username, password = auth.split(':', 1)
+            else:
+                username = password = None
             generic_parts['username'] = username
             generic_parts['password'] = password
             generic_parts['hostname'] = hostname
@@ -390,7 +404,7 @@ def generic_urlparse(parts):
         except:
             generic_parts['username'] = None
             generic_parts['password'] = None
-            generic_parts['hostname'] = None
+            generic_parts['hostname'] = parts[1]
             generic_parts['port']     = None
     return generic_parts
 
@@ -403,7 +417,7 @@ class RequestWithMethod(urllib2.Request):
     def __init__(self, url, method, data=None, headers=None):
         if headers is None:
             headers = {}
-        self._method = method
+        self._method = method.upper()
         urllib2.Request.__init__(self, url, data, headers)
 
     def get_method(self):
@@ -413,6 +427,55 @@ class RequestWithMethod(urllib2.Request):
             return urllib2.Request.get_method(self)
 
 
+def RedirectHandlerFactory(follow_redirects=None, validate_certs=True):
+    """This is a class factory that closes over the value of
+    ``follow_redirects`` so that the RedirectHandler class has access to
+    that value without having to use globals, and potentially cause problems
+    where ``open_url`` or ``fetch_url`` are used multiple times in a module.
+    """
+
+    class RedirectHandler(urllib2.HTTPRedirectHandler):
+        """This is an implementation of a RedirectHandler to match the
+        functionality provided by httplib2. It will utilize the value of
+        ``follow_redirects`` that is passed into ``RedirectHandlerFactory``
+        to determine how redirects should be handled in urllib2.
+        """
+
+        def redirect_request(self, req, fp, code, msg, hdrs, newurl):
+            handler = maybe_add_ssl_handler(newurl, validate_certs)
+            if handler:
+                urllib2._opener.add_handler(handler)
+
+            if follow_redirects == 'urllib2':
+                return urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, hdrs, newurl)
+            elif follow_redirects in ['no', 'none', False]:
+                raise urllib2.HTTPError(newurl, code, msg, hdrs, fp)
+
+            do_redirect = False
+            if follow_redirects in ['all', 'yes', True]:
+                do_redirect = (code >= 300 and code < 400)
+
+            elif follow_redirects == 'safe':
+                m = req.get_method()
+                do_redirect = (code >= 300 and code < 400 and m in ('GET', 'HEAD'))
+
+            if do_redirect:
+                # be conciliant with URIs containing a space
+                newurl = newurl.replace(' ', '%20')
+                newheaders = dict((k,v) for k,v in req.headers.items()
+                                  if k.lower() not in ("content-length", "content-type")
+                                 )
+                return urllib2.Request(newurl,
+                                       headers=newheaders,
+                                       origin_req_host=req.get_origin_req_host(),
+                                       unverifiable=True)
+            else:
+                raise urllib2.HTTPError(req.get_full_url(), code, msg, hdrs,
+                                        fp)
+
+    return RedirectHandler
+
+
 class SSLValidationHandler(urllib2.BaseHandler):
     '''
     A custom handler class for SSL validation.
@@ -532,7 +595,8 @@ class SSLValidationHandler(urllib2.BaseHandler):
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             if https_proxy:
                 proxy_parts = generic_urlparse(urlparse.urlparse(https_proxy))
-                s.connect((proxy_parts.get('hostname'), proxy_parts.get('port')))
+                port = proxy_parts.get('port') or 443
+                s.connect((proxy_parts.get('hostname'), port))
                 if proxy_parts.get('scheme') == 'http':
                     s.sendall(self.CONNECT_COMMAND % (self.hostname, self.port))
                     if proxy_parts.get('username'):
@@ -542,7 +606,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     connect_result = s.recv(4096)
                     self.validate_proxy_response(connect_result)
                     if context:
-                        ssl_s = context.wrap_socket(s, server_hostname=proxy_parts.get('hostname'))
+                        ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
                     else:
                         ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
                         match_hostname(ssl_s.getpeercert(), self.hostname)
@@ -586,14 +650,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
 
     https_request = http_request
 
-# Rewrite of fetch_url to not require the module environment
-def open_url(url, data=None, headers=None, method=None, use_proxy=True,
-        force=False, last_mod_time=None, timeout=10, validate_certs=True,
-        url_username=None, url_password=None, http_agent=None, force_basic_auth=False):
-    '''
-    Fetches a file from an HTTP/FTP server using urllib2
-    '''
-    handlers = []
+def maybe_add_ssl_handler(url, validate_certs):
     # FIXME: change the following to use the generic_urlparse function
     #        to remove the indexed references for 'parsed'
     parsed = urlparse.urlparse(url)
@@ -613,9 +670,24 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
             port = 443
         # create the SSL validation handler and
         # add it to the list of handlers
-        ssl_handler = SSLValidationHandler(hostname, port)
+        return SSLValidationHandler(hostname, port)
+
+# Rewrite of fetch_url to not require the module environment
+def open_url(url, data=None, headers=None, method=None, use_proxy=True,
+        force=False, last_mod_time=None, timeout=10, validate_certs=True,
+        url_username=None, url_password=None, http_agent=None,
+        force_basic_auth=False, follow_redirects='urllib2'):
+    '''
+    Fetches a file from an HTTP/FTP server using urllib2
+    '''
+    handlers = []
+    ssl_handler = maybe_add_ssl_handler(url, validate_certs)
+    if ssl_handler:
         handlers.append(ssl_handler)
 
+    # FIXME: change the following to use the generic_urlparse function
+    #        to remove the indexed references for 'parsed'
+    parsed = urlparse.urlparse(url)
     if parsed[0] != 'ftp':
         username = url_username
 
@@ -661,10 +733,13 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
         handlers.append(proxyhandler)
 
     # pre-2.6 versions of python cannot use the custom https
-    # handler, since the socket class is lacking this method
-    if hasattr(socket, 'create_connection'):
+    # handler, since the socket class is lacking create_connection.
+    # Some python builds lack HTTPS support.
+    if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
         handlers.append(CustomHTTPSHandler)
 
+    handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
+
     opener = urllib2.build_opener(*handlers)
     urllib2.install_opener(opener)
 
@@ -752,26 +827,29 @@ def fetch_url(module, url, data=None, headers=None, method=None,
     http_agent = module.params.get('http_agent', None)
     force_basic_auth = module.params.get('force_basic_auth', '')
 
+    follow_redirects = module.params.get('follow_redirects', 'urllib2')
+
     r = None
     info = dict(url=url)
     try:
         r = open_url(url, data=data, headers=headers, method=method,
-                use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout,
-                validate_certs=validate_certs, url_username=username,
-                url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth)
+                     use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout,
+                     validate_certs=validate_certs, url_username=username,
+                     url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
+                     follow_redirects=follow_redirects)
         info.update(r.info())
         info['url'] = r.geturl()  # The URL goes in too, because of redirects.
         info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
     except NoSSLError, e:
         distribution = get_distribution()
-        if distribution.lower() == 'redhat':
+        if distribution is not None and distribution.lower() == 'redhat':
             module.fail_json(msg='%s. You can also install python-ssl from EPEL' % str(e))
         else:
             module.fail_json(msg='%s' % str(e))
     except (ConnectionError, ValueError), e:
         module.fail_json(msg=str(e))
     except urllib2.HTTPError, e:
-        info.update(dict(msg=str(e), status=e.code))
+        info.update(dict(msg=str(e), status=e.code, **e.info()))
     except urllib2.URLError, e:
         code = int(getattr(e, 'code', -1))
         info.update(dict(msg="Request failed: %s" % str(e), status=code))

lib/ansible/module_utils/vca.py

@@ -35,8 +35,8 @@ class VcaError(Exception):
 
 def vca_argument_spec():
     return dict(
-        username=dict(),
-        password=dict(),
+        username=dict(type='str', aliases=['user'], required=True),
+        password=dict(type='str', aliases=['pass','passwd'], required=True, no_log=True),
         org=dict(),
         service_id=dict(),
         instance_id=dict(),
@@ -108,7 +108,10 @@ class VcaAnsibleModule(AnsibleModule):
 
     def create_instance(self):
         service_type = self.params.get('service_type', DEFAULT_SERVICE_TYPE)
-        host = self.params.get('host', LOGIN_HOST.get('service_type'))
+        if service_type == 'vcd': 
+            host = self.params['host']
+        else:
+            host = LOGIN_HOST[service_type]
         username = self.params['username']
 
         version = self.params.get('api_version')

lib/ansible/module_utils/vmware.py

@@ -21,6 +21,7 @@
 try:
     import atexit
     import time
+    import ssl
     # requests is required for exception handling of the ConnectionError
     import requests
     from pyVim import connect
@@ -98,12 +99,30 @@ def find_hostsystem_by_name(content, hostname):
     return None
 
 
+def find_vm_by_name(content, vm_name):
+
+    vms = get_all_objs(content, [vim.VirtualMachine])
+    for vm in vms:
+        if vm.name == vm_name:
+            return vm
+    return None
+
+
+def find_host_portgroup_by_name(host, portgroup_name):
+
+    for portgroup in host.config.network.portgroup:
+        if portgroup.spec.name == portgroup_name:
+            return portgroup
+    return None
+
+
 def vmware_argument_spec():
 
     return dict(
         hostname=dict(type='str', required=True),
         username=dict(type='str', aliases=['user', 'admin'], required=True),
         password=dict(type='str', aliases=['pass', 'pwd'], required=True, no_log=True),
+        validate_certs=dict(type='bool', required=False, default=True),
     )
 
 
@@ -112,21 +131,29 @@ def connect_to_api(module, disconnect_atexit=True):
     hostname = module.params['hostname']
     username = module.params['username']
     password = module.params['password']
+    validate_certs = module.params['validate_certs']
+
+    if validate_certs and not hasattr(ssl, 'SSLContext'):
+        module.fail_json(msg='pyVim does not support changing verification mode with python < 2.7.9. Either update python or or use validate_certs=false')
+
     try:
         service_instance = connect.SmartConnect(host=hostname, user=username, pwd=password)
-
-        # Disabling atexit should be used in special cases only.
-        # Such as IP change of the ESXi host which removes the connection anyway.
-        # Also removal significantly speeds up the return of the module
-
-        if disconnect_atexit:
-            atexit.register(connect.Disconnect, service_instance)
-        return service_instance.RetrieveContent()
     except vim.fault.InvalidLogin, invalid_login:
         module.fail_json(msg=invalid_login.msg, apierror=str(invalid_login))
     except requests.ConnectionError, connection_error:
-        module.fail_json(msg="Unable to connect to vCenter or ESXi API on TCP/443.", apierror=str(connection_error))
+        if '[SSL: CERTIFICATE_VERIFY_FAILED]' in str(connection_error) and not validate_certs:
+            context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            context.verify_mode = ssl.CERT_NONE
+            service_instance = connect.SmartConnect(host=hostname, user=username, pwd=password, sslContext=context)
+        else:
+            module.fail_json(msg="Unable to connect to vCenter or ESXi API on TCP/443.", apierror=str(connection_error))
 
+    # Disabling atexit should be used in special cases only.
+    # Such as IP change of the ESXi host which removes the connection anyway.
+    # Also removal significantly speeds up the return of the module
+    if disconnect_atexit:
+        atexit.register(connect.Disconnect, service_instance)
+    return service_instance.RetrieveContent()
 
 def get_all_objs(content, vimtype):
 

lib/ansible/parsing/dataloader.py

@@ -20,12 +20,10 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import copy
-import json
 import os
-import stat
+import json
 import subprocess
-
-from yaml import load, YAMLError
+from yaml import YAMLError
 from ansible.compat.six import text_type, string_types
 
 from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
@@ -36,7 +34,7 @@ from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
 from ansible.module_utils.basic import is_executable
 from ansible.utils.path import unfrackpath
-from ansible.utils.unicode import to_unicode
+from ansible.utils.unicode import to_unicode, to_bytes
 
 class DataLoader():
 
@@ -73,31 +71,30 @@ class DataLoader():
         Creates a python datastructure from the given data, which can be either
         a JSON or YAML string.
         '''
-
+        new_data = None
         try:
             # we first try to load this data as JSON
-            return json.loads(data)
+            new_data = json.loads(data)
         except:
-            # if loading JSON failed for any reason, we go ahead
-            # and try to parse it as YAML instead
-
+            # must not be JSON, let the rest try
             if isinstance(data, AnsibleUnicode):
                 # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
                 # they are unable to cope with our subclass.
                 # Unwrap and re-wrap the unicode so we can keep track of line
                 # numbers
-                new_data = text_type(data)
+                in_data = text_type(data)
             else:
-                new_data = data
+                in_data = data
             try:
-                new_data = self._safe_load(new_data, file_name=file_name)
+                new_data = self._safe_load(in_data, file_name=file_name)
             except YAMLError as yaml_exc:
                 self._handle_error(yaml_exc, file_name, show_content)
 
             if isinstance(data, AnsibleUnicode):
                 new_data = AnsibleUnicode(new_data)
                 new_data.ansible_pos = data.ansible_pos
-            return new_data
+
+        return new_data
 
     def load_from_file(self, file_name):
         ''' Loads data from a file, which can contain either JSON or YAML.  '''
@@ -121,15 +118,15 @@ class DataLoader():
 
     def path_exists(self, path):
         path = self.path_dwim(path)
-        return os.path.exists(path)
+        return os.path.exists(to_bytes(path, errors='strict'))
 
     def is_file(self, path):
         path = self.path_dwim(path)
-        return os.path.isfile(path) or path == os.devnull
+        return os.path.isfile(to_bytes(path, errors='strict')) or path == os.devnull
 
     def is_directory(self, path):
         path = self.path_dwim(path)
-        return os.path.isdir(path)
+        return os.path.isdir(to_bytes(path, errors='strict'))
 
     def list_directory(self, path):
         path = self.path_dwim(path)
@@ -147,7 +144,10 @@ class DataLoader():
         try:
             return loader.get_single_data()
         finally:
-            loader.dispose()
+            try:
+                loader.dispose()
+            except AttributeError:
+                pass # older versions of yaml don't have dispose function, ignore
 
     def _get_file_contents(self, file_name):
         '''
@@ -206,13 +206,15 @@ class DataLoader():
         '''
 
         given = unquote(given)
+        given = to_unicode(given, errors='strict')
 
-        if given.startswith("/"):
+        if given.startswith(u"/"):
             return os.path.abspath(given)
-        elif given.startswith("~"):
+        elif given.startswith(u"~"):
             return os.path.abspath(os.path.expanduser(given))
         else:
-            return os.path.abspath(os.path.join(self._basedir, given))
+            basedir = to_unicode(self._basedir, errors='strict')
+            return os.path.abspath(os.path.join(basedir, given))
 
     def path_dwim_relative(self, path, dirname, source):
         '''
@@ -236,8 +238,8 @@ class DataLoader():
             basedir = unfrackpath(path)
 
             # is it a role and if so make sure you get correct base path
-            if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \
-                or os.path.exists(os.path.join(path,'tasks/main.yml')):
+            if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='strict')) \
+                or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='strict')):
                 isrole = True
                 if path.endswith('tasks'):
                     basedir = unfrackpath(os.path.dirname(path))
@@ -260,7 +262,7 @@ class DataLoader():
             search.append(self.path_dwim(source))
 
         for candidate in search:
-            if os.path.exists(candidate):
+            if os.path.exists(to_bytes(candidate, errors='strict')):
                 break
 
         return candidate
@@ -271,8 +273,8 @@ class DataLoader():
         retrieve password from STDOUT
         """
 
-        this_path = os.path.realpath(os.path.expanduser(vault_password_file))
-        if not os.path.exists(this_path):
+        this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='strict'))
+        if not os.path.exists(to_bytes(this_path, errors='strict')):
             raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
 
         if self.is_executable(this_path):

lib/ansible/parsing/mod_args.py

@@ -137,7 +137,16 @@ class ModuleArgsParser:
         # than those which may be parsed/normalized next
         final_args = dict()
         if additional_args:
-            final_args.update(additional_args)
+            if isinstance(additional_args, string_types):
+                templar = Templar(loader=None)
+                if templar._contains_vars(additional_args):
+                    final_args['_variable_params'] = additional_args
+                else:
+                    raise AnsibleParserError("Complex args containing variables cannot use bare variables, and must use the full variable style ('{{var_name}}')")
+            elif isinstance(additional_args, dict):
+                final_args.update(additional_args)
+            else:
+                raise AnsibleParserError('Complex args must be a dictionary or variable string ("{{var}}").')
 
         # how we normalize depends if we figured out what the module name is
         # yet.  If we have already figured it out, it's an 'old style' invocation.
@@ -213,18 +222,21 @@ class ModuleArgsParser:
         action = None
         args = None
 
+        actions_allowing_raw = ('command', 'shell', 'script', 'raw')
         if isinstance(thing, dict):
             # form is like:  copy: { src: 'a', dest: 'b' } ... common for structured (aka "complex") args
             thing = thing.copy()
             if 'module' in thing:
-                action = thing['module']
+                action, module_args = self._split_module_string(thing['module'])
                 args = thing.copy()
+                check_raw = action in actions_allowing_raw
+                args.update(parse_kv(module_args, check_raw=check_raw))
                 del args['module']
 
         elif isinstance(thing, string_types):
             # form is like:  copy: src=a dest=b ... common shorthand throughout ansible
             (action, args) = self._split_module_string(thing)
-            check_raw = action in ('command', 'shell', 'script', 'raw')
+            check_raw = action in actions_allowing_raw
             args = parse_kv(args, check_raw=check_raw)
 
         else:
@@ -289,7 +301,7 @@ class ModuleArgsParser:
                         obj=self._task_ds)
 
             else:
-                raise AnsibleParserError("no action detected in task", obj=self._task_ds)
+                raise AnsibleParserError("no action detected in task. This often indicates a misspelled module name, or incorrect module path.", obj=self._task_ds)
         elif args.get('_raw_params', '') != '' and action not in RAW_PARAM_MODULES:
             templar = Templar(loader=None)
             raw_params = args.pop('_raw_params')

lib/ansible/parsing/splitter.py

@@ -22,7 +22,7 @@ __metaclass__ = type
 import re
 import codecs
 
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleParserError
 from ansible.parsing.quoting import unquote
 
 # Decode escapes adapted from rspeer's answer here:
@@ -60,13 +60,13 @@ def parse_kv(args, check_raw=False):
             vargs = split_args(args)
         except ValueError as ve:
             if 'no closing quotation' in str(ve).lower():
-                raise AnsibleError("error parsing argument string, try quoting the entire line.")
+                raise AnsibleParsingError("error parsing argument string, try quoting the entire line.")
             else:
                 raise
 
         raw_params = []
-        for x in vargs:
-            x = _decode_escapes(x)
+        for orig_x in vargs:
+            x = _decode_escapes(orig_x)
             if "=" in x:
                 pos = 0
                 try:
@@ -86,11 +86,11 @@ def parse_kv(args, check_raw=False):
                 # FIXME: make the retrieval of this list of shell/command
                 #        options a function, so the list is centralized
                 if check_raw and k not in ('creates', 'removes', 'chdir', 'executable', 'warn'):
-                    raw_params.append(x)
+                    raw_params.append(orig_x)
                 else:
                     options[k.strip()] = unquote(v.strip())
             else:
-                raw_params.append(x)
+                raw_params.append(orig_x)
 
         # recombine the free-form params, if any were found, and assign
         # them to a special option for use later by the shell/command module
@@ -256,6 +256,6 @@ def split_args(args):
     # If we're done and things are not at zero depth or we're still inside quotes,
     # raise an error to indicate that the args were unbalanced
     if print_depth or block_depth or comment_depth or inside_quotes:
-        raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")
+        raise AnsibleParserError("failed at splitting arguments, either an unbalanced jinja2 block or quotes")
 
     return params

lib/ansible/parsing/utils/addresses.py

@@ -20,6 +20,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import re
+from ansible.errors import AnsibleParserError, AnsibleError
 
 # Components that match a numeric or alphanumeric begin:end or begin:end:step
 # range expression inside square brackets.
@@ -162,6 +163,7 @@ patterns = {
             $
         '''.format(label=label), re.X|re.I|re.UNICODE
     ),
+
 }
 
 def parse_address(address, allow_ranges=False):
@@ -183,8 +185,8 @@ def parse_address(address, allow_ranges=False):
     # First, we extract the port number if one is specified.
 
     port = None
-    for type in ['bracketed_hostport', 'hostport']:
-        m = patterns[type].match(address)
+    for matching in ['bracketed_hostport', 'hostport']:
+        m = patterns[matching].match(address)
         if m:
             (address, port) = m.groups()
             port = int(port)
@@ -194,22 +196,20 @@ def parse_address(address, allow_ranges=False):
     # numeric ranges, or a hostname with alphanumeric ranges.
 
     host = None
-    for type in ['ipv4', 'ipv6', 'hostname']:
-        m = patterns[type].match(address)
+    for matching in ['ipv4', 'ipv6', 'hostname']:
+        m = patterns[matching].match(address)
         if m:
             host = address
             continue
 
     # If it isn't any of the above, we don't understand it.
-
     if not host:
-        return (None, None)
-
-    # If we get to this point, we know that any included ranges are valid. If
-    # the caller is prepared to handle them, all is well. Otherwise we treat
-    # it as a parse failure.
+        raise AnsibleError("Not a valid network hostname: %s" % address)
 
+    # If we get to this point, we know that any included ranges are valid.
+    # If the caller is prepared to handle them, all is well.
+    # Otherwise we treat it as a parse failure.
     if not allow_ranges and '[' in host:
-        return (None, None)
+        raise AnsibleParserError("Detected range in host but was asked to ignore ranges")
 
     return (host, port)

lib/ansible/parsing/utils/jsonify.py

@@ -29,17 +29,13 @@ def jsonify(result, format=False):
 
     if result is None:
         return "{}"
-    result2 = result.copy()
-    for key, value in result2.items():
-        if type(value) is str:
-            result2[key] = value.decode('utf-8', 'ignore')
 
     indent = None
     if format:
         indent = 4
 
     try:
-        return json.dumps(result2, sort_keys=True, indent=indent, ensure_ascii=False)
+        return json.dumps(result, sort_keys=True, indent=indent, ensure_ascii=False)
     except UnicodeDecodeError:
-        return json.dumps(result2, sort_keys=True, indent=indent)
+        return json.dumps(result, sort_keys=True, indent=indent)
 

lib/ansible/parsing/vault/__init__.py

@@ -22,6 +22,7 @@ import shlex
 import shutil
 import sys
 import tempfile
+import random
 from io import BytesIO
 from subprocess import call
 from ansible.errors import AnsibleError
@@ -70,7 +71,7 @@ try:
 except ImportError:
     pass
 
-from ansible.compat.six import PY3, byte2int
+from ansible.compat.six import PY3
 from ansible.utils.unicode import to_unicode, to_bytes
 
 HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC
@@ -220,21 +221,95 @@ class VaultEditor:
     def __init__(self, password):
         self.vault = VaultLib(password)
 
+    def _shred_file_custom(self, tmp_path):
+        """"Destroy a file, when shred (core-utils) is not available
+
+        Unix `shred' destroys files "so that they can be recovered only with great difficulty with
+        specialised hardware, if at all". It is based on the method from the paper
+        "Secure Deletion of Data from Magnetic and Solid-State Memory",
+        Proceedings of the Sixth USENIX Security Symposium (San Jose, California, July 22-25, 1996).
+
+        We do not go to that length to re-implement shred in Python; instead, overwriting with a block
+        of random data should suffice.
+
+        See https://github.com/ansible/ansible/pull/13700 .
+        """
+
+        file_len = os.path.getsize(tmp_path)
+
+        if file_len > 0: # avoid work when file was empty
+            max_chunk_len = min(1024*1024*2, file_len)
+
+            passes = 3
+            with open(tmp_path,  "wb") as fh:
+                for _ in range(passes):
+                    fh.seek(0,  0)
+                    # get a random chunk of data, each pass with other length
+                    chunk_len = random.randint(max_chunk_len//2, max_chunk_len)
+                    data = os.urandom(chunk_len)
+
+                    for _ in range(0, file_len // chunk_len):
+                        fh.write(data)
+                    fh.write(data[:file_len % chunk_len])
+
+                    assert(fh.tell() == file_len) # FIXME remove this assert once we have unittests to check its accuracy
+                    os.fsync(fh)
+
+
+    def _shred_file(self, tmp_path):
+        """Securely destroy a decrypted file
+
+        Note standard limitations of GNU shred apply (For flash, overwriting would have no effect
+        due to wear leveling; for other storage systems, the async kernel->filesystem->disk calls never
+        guarantee data hits the disk; etc). Furthermore, if your tmp dirs is on tmpfs (ramdisks),
+        it is a non-issue.
+
+        Nevertheless, some form of overwriting the data (instead of just removing the fs index entry) is
+        a good idea. If shred is not available (e.g. on windows, or no core-utils installed), fall back on
+        a custom shredding method.
+        """
+
+        if not os.path.isfile(tmp_path):
+            # file is already gone
+            return
+
+        try:
+            r = call(['shred', tmp_path])
+        except (OSError, ValueError):
+            # shred is not available on this system, or some other error occured.
+            # ValueError caught because OS X El Capitan is raising an
+            # exception big enough to hit a limit in python2-2.7.11 and below.
+            # Symptom is ValueError: insecure pickle when shred is not
+            # installed there.
+            r = 1
+
+        if r != 0:
+            # we could not successfully execute unix shred; therefore, do custom shred.
+            self._shred_file_custom(tmp_path)
+
+        os.remove(tmp_path)
+
     def _edit_file_helper(self, filename, existing_data=None, force_save=False):
 
         # Create a tempfile
         _, tmp_path = tempfile.mkstemp()
 
         if existing_data:
-            self.write_data(existing_data, tmp_path)
+            self.write_data(existing_data, tmp_path, shred=False)
 
         # drop the user into an editor on the tmp file
-        call(self._editor_shell_command(tmp_path))
+        try:
+            call(self._editor_shell_command(tmp_path))
+        except:
+            # whatever happens, destroy the decrypted file
+            self._shred_file(tmp_path)
+            raise
+
         tmpdata = self.read_data(tmp_path)
 
         # Do nothing if the content has not changed
         if existing_data == tmpdata and not force_save:
-            os.remove(tmp_path)
+            self._shred_file(tmp_path)
             return
 
         # encrypt new data and write out to tmp
@@ -257,8 +332,11 @@ class VaultEditor:
         check_prereqs()
 
         ciphertext = self.read_data(filename)
-        plaintext = self.vault.decrypt(ciphertext)
-        self.write_data(plaintext, output_file or filename)
+        try:
+            plaintext = self.vault.decrypt(ciphertext)
+        except AnsibleError as e:
+            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))
+        self.write_data(plaintext, output_file or filename, shred=False)
 
     def create_file(self, filename):
         """ create a new encrypted file """
@@ -277,7 +355,10 @@ class VaultEditor:
         check_prereqs()
 
         ciphertext = self.read_data(filename)
-        plaintext = self.vault.decrypt(ciphertext)
+        try:
+            plaintext = self.vault.decrypt(ciphertext)
+        except AnsibleError as e:
+            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))
 
         if self.vault.cipher_name not in CIPHER_WRITE_WHITELIST:
             # we want to get rid of files encrypted with the AES cipher
@@ -288,9 +369,12 @@ class VaultEditor:
     def plaintext(self, filename):
 
         check_prereqs()
-
         ciphertext = self.read_data(filename)
-        plaintext = self.vault.decrypt(ciphertext)
+
+        try:
+            plaintext = self.vault.decrypt(ciphertext)
+        except AnsibleError as e:
+            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))
 
         return plaintext
 
@@ -300,7 +384,10 @@ class VaultEditor:
 
         prev = os.stat(filename)
         ciphertext = self.read_data(filename)
-        plaintext = self.vault.decrypt(ciphertext)
+        try:
+            plaintext = self.vault.decrypt(ciphertext)
+        except AnsibleError as e:
+            raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename)))
 
         new_vault = VaultLib(new_password)
         new_ciphertext = new_vault.encrypt(plaintext)
@@ -312,6 +399,7 @@ class VaultEditor:
         os.chown(filename, prev.st_uid, prev.st_gid)
 
     def read_data(self, filename):
+
         try:
             if filename == '-':
                 data = sys.stdin.read()
@@ -323,13 +411,21 @@ class VaultEditor:
 
         return data
 
-    def write_data(self, data, filename):
+    def write_data(self, data, filename, shred=True):
+        """write data to given path
+        
+        if shred==True, make sure that the original data is first shredded so 
+        that is cannot be recovered
+        """
         bytes = to_bytes(data, errors='strict')
         if filename == '-':
             sys.stdout.write(bytes)
         else:
             if os.path.isfile(filename):
-                os.remove(filename)
+                if shred:
+                    self._shred_file(filename)
+                else:
+                    os.remove(filename)
             with open(filename, "wb") as fh:
                 fh.write(bytes)
 
@@ -338,6 +434,7 @@ class VaultEditor:
         # overwrite dest with src
         if os.path.isfile(dest):
             prev = os.stat(dest)
+            # old file 'dest' was encrypted, no need to _shred_file
             os.remove(dest)
         shutil.move(src, dest)
 
@@ -391,7 +488,7 @@ class VaultFile(object):
             this_vault = VaultLib(self.password)
             dec_data = this_vault.decrypt(tmpdata)
             if dec_data is None:
-                raise AnsibleError("Decryption failed")
+                raise AnsibleError("Failed to decrypt: %s" % self.filename)
             else:
                 self.tmpfile.write(dec_data)
                 return self.tmpfile

lib/ansible/parsing/yaml/constructor.py

@@ -22,6 +22,7 @@ __metaclass__ = type
 from yaml.constructor import Constructor, ConstructorError
 from yaml.nodes import MappingNode
 from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
+from ansible.vars.unsafe_proxy import wrap_var
 
 try:
     from __main__ import display
@@ -65,14 +66,14 @@ class AnsibleConstructor(Constructor):
                         "found unacceptable key (%s)" % exc, key_node.start_mark)
 
             if key in mapping:
-                display.warning('While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).  Using last defined value only.'.format(key, *mapping.ansible_pos))
+                display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).  Using last defined value only.'.format(key, *mapping.ansible_pos))
 
             value = self.construct_object(value_node, deep=deep)
             mapping[key] = value
 
         return mapping
 
-    def construct_yaml_str(self, node):
+    def construct_yaml_str(self, node, unsafe=False):
         # Override the default string handling function
         # to always return unicode objects
         value = self.construct_scalar(node)
@@ -80,6 +81,9 @@ class AnsibleConstructor(Constructor):
 
         ret.ansible_pos = self._node_position_info(node)
 
+        if unsafe:
+            ret = wrap_var(ret)
+
         return ret
 
     def construct_yaml_seq(self, node):
@@ -88,6 +92,9 @@ class AnsibleConstructor(Constructor):
         data.extend(self.construct_sequence(node))
         data.ansible_pos = self._node_position_info(node)
 
+    def construct_yaml_unsafe(self, node):
+        return self.construct_yaml_str(node, unsafe=True)
+
     def _node_position_info(self, node):
         # the line number where the previous token has ended (plus empty lines)
         # Add one so that the first line is line 1 rather than line 0
@@ -121,3 +128,7 @@ AnsibleConstructor.add_constructor(
 AnsibleConstructor.add_constructor(
     u'tag:yaml.org,2002:seq',
     AnsibleConstructor.construct_yaml_seq)
+
+AnsibleConstructor.add_constructor(
+    u'!unsafe',
+    AnsibleConstructor.construct_yaml_unsafe)

lib/ansible/parsing/yaml/dumper.py

@@ -22,7 +22,7 @@ __metaclass__ = type
 import yaml
 from ansible.compat.six import PY3
 
-from ansible.parsing.yaml.objects import AnsibleUnicode
+from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping
 from ansible.vars.hostvars import HostVars
 
 class AnsibleDumper(yaml.SafeDumper):
@@ -50,3 +50,13 @@ AnsibleDumper.add_representer(
     represent_hostvars,
 )
 
+AnsibleDumper.add_representer(
+    AnsibleSequence,
+    yaml.representer.SafeRepresenter.represent_list,
+)
+
+AnsibleDumper.add_representer(
+    AnsibleMapping,
+    yaml.representer.SafeRepresenter.represent_dict,
+)
+

lib/ansible/playbook/__init__.py

@@ -25,6 +25,7 @@ from ansible.errors import AnsibleParserError
 from ansible.playbook.play import Play
 from ansible.playbook.playbook_include import PlaybookInclude
 from ansible.plugins import get_all_plugin_loaders
+from ansible import constants as C
 
 try:
     from __main__ import display
@@ -87,7 +88,7 @@ class Playbook:
                 if pb is not None:
                     self._entries.extend(pb._entries)
                 else:
-                    display.display("skipping playbook include '%s' due to conditional test failure" % entry.get('include', entry), color='cyan')
+                    display.display("skipping playbook include '%s' due to conditional test failure" % entry.get('include', entry), color=C.COLOR_SKIP)
             else:
                 entry_obj = Play.load(entry, variable_manager=variable_manager, loader=self._loader)
                 self._entries.append(entry_obj)

lib/ansible/playbook/attribute.py

@@ -19,6 +19,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from copy import deepcopy
 
 class Attribute:
 
@@ -32,6 +33,11 @@ class Attribute:
         self.priority = priority
         self.always_post_validate = always_post_validate
 
+        if default is not None and self.isa in ('list', 'dict', 'set'):
+            self.default = deepcopy(default)
+        else:
+            self.default = default
+
     def __eq__(self, other):
         return other.priority == self.priority
 

lib/ansible/playbook/base.py

@@ -153,7 +153,7 @@ class Base:
             setattr(Base, name, property(getter, setter, deleter))
 
             # Place the value into the instance so that the property can
-            # process and hold that value/
+            # process and hold that value.
             setattr(self, name, value.default)
 
     def preprocess_data(self, ds):
@@ -267,6 +267,8 @@ class Base:
         new_me._loader           = self._loader
         new_me._variable_manager = self._variable_manager
 
+        new_me._uuid = self._uuid
+
         # if the ds value was set on the object, copy it to the new copy too
         if hasattr(self, '_ds'):
             new_me._ds = self._ds
@@ -332,7 +334,10 @@ class Base:
                         if value is None:
                             value = []
                         elif not isinstance(value, list):
-                            value = [ value ]
+                            if isinstance(value, string_types):
+                                value = value.split(',')
+                            else:
+                                value = [ value ]
                         if attribute.listof is not None:
                             for item in value:
                                 if not isinstance(item, attribute.listof):
@@ -344,11 +349,15 @@ class Base:
                     elif attribute.isa == 'set':
                         if value is None:
                             value = set()
-                        else:
-                            if not isinstance(value, (list, set)):
+                        elif not isinstance(value, (list, set)):
+                            if isinstance(value, string_types):
+                                value = value.split(',')
+                            else:
+                                # Making a list like this handles strings of
+                                # text and bytes properly
                                 value = [ value ]
-                            if not isinstance(value, set):
-                                value = set(value)
+                        if not isinstance(value, set):
+                            value = set(value)
                     elif attribute.isa == 'dict':
                         if value is None:
                             value = dict()
@@ -414,7 +423,7 @@ class Base:
         def _validate_variable_keys(ds):
             for key in ds:
                 if not isidentifier(key):
-                    raise TypeError("%s is not a valid variable name" % key)
+                    raise TypeError("'%s' is not a valid variable name" % key)
 
         try:
             if isinstance(ds, dict):

lib/ansible/playbook/become.py

@@ -31,7 +31,7 @@ except ImportError:
 
 class Become:
 
-    # Privlege escalation
+    # Privilege escalation
     _become              = FieldAttribute(isa='bool')
     _become_method       = FieldAttribute(isa='string')
     _become_user         = FieldAttribute(isa='string')
@@ -60,7 +60,7 @@ class Become:
 
         This is called from the Base object's preprocess_data() method which
         in turn is called pretty much anytime any sort of playbook object
-        (plays, tasks, blocks, etc) are created.
+        (plays, tasks, blocks, etc) is created.
         """
 
         self._detect_privilege_escalation_conflict(ds)
@@ -90,16 +90,17 @@ class Become:
 
             display.deprecated("Instead of su/su_user, use become/become_user and set become_method to 'su' (default is sudo)")
 
-        # if we are becoming someone else, but some fields are unset,
-        # make sure they're initialized to the default config values
-        if ds.get('become', False):
-            if ds.get('become_method', None) is None:
-                ds['become_method'] = C.DEFAULT_BECOME_METHOD
-            if ds.get('become_user', None) is None:
-                ds['become_user'] = C.DEFAULT_BECOME_USER
-
         return ds
 
+    def set_become_defaults(self, become, become_method, become_user):
+        ''' if we are becoming someone else, but some fields are unset,
+            make sure they're initialized to the default config values  '''
+        if become:
+            if become_method is None:
+                become_method = C.DEFAULT_BECOME_METHOD
+            if become_user is None:
+                become_user = C.DEFAULT_BECOME_USER
+
     def _get_attr_become(self):
         '''
         Override for the 'become' getattr fetcher, used from Base.

lib/ansible/playbook/block.py

@@ -34,6 +34,8 @@ class Block(Base, Become, Conditional, Taggable):
     _rescue = FieldAttribute(isa='list', default=[])
     _always = FieldAttribute(isa='list', default=[])
     _delegate_to = FieldAttribute(isa='list')
+    _delegate_facts = FieldAttribute(isa='bool', default=False)
+    _any_errors_fatal = FieldAttribute(isa='bool')
 
     # for future consideration? this would be functionally
     # similar to the 'else' clause for exceptions
@@ -42,11 +44,16 @@ class Block(Base, Become, Conditional, Taggable):
     def __init__(self, play=None, parent_block=None, role=None, task_include=None, use_handlers=False, implicit=False):
         self._play         = play
         self._role         = role
-        self._task_include = task_include
-        self._parent_block = parent_block
+        self._task_include = None
+        self._parent_block = None
+        self._dep_chain    = None
         self._use_handlers = use_handlers
         self._implicit     = implicit
-        self._dep_chain    = []
+
+        if task_include:
+            self._task_include = task_include
+        elif parent_block:
+            self._parent_block = parent_block
 
         super(Block, self).__init__()
 
@@ -142,6 +149,17 @@ class Block(Base, Become, Conditional, Taggable):
         except AssertionError:
             raise AnsibleParserError("A malformed block was encountered.", obj=self._ds)
 
+    def get_dep_chain(self):
+        if self._dep_chain is None:
+            if self._parent_block:
+                return self._parent_block.get_dep_chain()
+            elif self._task_include:
+                return self._task_include._block.get_dep_chain()
+            else:
+                return None
+        else:
+            return self._dep_chain[:]
+
     def copy(self, exclude_parent=False, exclude_tasks=False):
         def _dupe_task_list(task_list, new_block):
             new_task_list = []
@@ -158,7 +176,9 @@ class Block(Base, Become, Conditional, Taggable):
         new_me = super(Block, self).copy()
         new_me._play         = self._play
         new_me._use_handlers = self._use_handlers
-        new_me._dep_chain    = self._dep_chain[:]
+
+        if self._dep_chain:
+            new_me._dep_chain = self._dep_chain[:]
 
         if not exclude_tasks:
             new_me.block  = _dupe_task_list(self.block or [], new_me)
@@ -175,7 +195,8 @@ class Block(Base, Become, Conditional, Taggable):
 
         new_me._task_include = None
         if self._task_include:
-            new_me._task_include = self._task_include.copy()
+            new_me._task_include = self._task_include.copy(exclude_block=True)
+            new_me._task_include._block = self._task_include._block.copy(exclude_tasks=True)
 
         return new_me
 
@@ -190,7 +211,7 @@ class Block(Base, Become, Conditional, Taggable):
             if attr not in ('block', 'rescue', 'always'):
                 data[attr] = getattr(self, attr)
 
-        data['dep_chain'] = self._dep_chain
+        data['dep_chain'] = self.get_dep_chain()
 
         if self._role is not None:
             data['role'] = self._role.serialize()
@@ -215,7 +236,7 @@ class Block(Base, Become, Conditional, Taggable):
             if attr in data and attr not in ('block', 'rescue', 'always'):
                 setattr(self, attr, data.get(attr))
 
-        self._dep_chain = data.get('dep_chain', [])
+        self._dep_chain = data.get('dep_chain', None)
 
         # if there was a serialized role, unpack it too
         role_data = data.get('role')
@@ -236,10 +257,12 @@ class Block(Base, Become, Conditional, Taggable):
             pb = Block()
             pb.deserialize(pb_data)
             self._parent_block = pb
+            self._dep_chain = self._parent_block.get_dep_chain()
 
     def evaluate_conditional(self, templar, all_vars):
-        if len(self._dep_chain):
-            for dep in self._dep_chain:
+        dep_chain = self.get_dep_chain()
+        if dep_chain:
+            for dep in dep_chain:
                 if not dep.evaluate_conditional(templar, all_vars):
                     return False
         if self._task_include is not None:
@@ -263,8 +286,10 @@ class Block(Base, Become, Conditional, Taggable):
         if self._task_include:
             self._task_include.set_loader(loader)
 
-        for dep in self._dep_chain:
-            dep.set_loader(loader)
+        dep_chain = self.get_dep_chain()
+        if dep_chain:
+            for dep in dep_chain:
+                dep.set_loader(loader)
 
     def _get_parent_attribute(self, attr, extend=False):
         '''
@@ -287,18 +312,18 @@ class Block(Base, Become, Conditional, Taggable):
                     value = self._extend_value(value, parent_value)
                 else:
                     value = parent_value
-            if self._role and (value is None or extend):
-                parent_value = getattr(self._role, attr)
+            if self._role and (value is None or extend) and hasattr(self._role, attr):
+                parent_value = getattr(self._role, attr, None)
                 if extend:
                     value = self._extend_value(value, parent_value)
                 else:
                     value = parent_value
 
-                if len(self._dep_chain) and (not value or extend):
-                    reverse_dep_chain = self._dep_chain[:]
-                    reverse_dep_chain.reverse()
-                    for dep in reverse_dep_chain:
-                        dep_value = getattr(dep, attr)
+                dep_chain = self.get_dep_chain()
+                if dep_chain and (value is None or extend):
+                    dep_chain.reverse()
+                    for dep in dep_chain:
+                        dep_value = getattr(dep, attr, None)
                         if extend:
                             value = self._extend_value(value, dep_value)
                         else:
@@ -306,14 +331,13 @@ class Block(Base, Become, Conditional, Taggable):
 
                         if value is not None and not extend:
                             break
-
-            if self._play and (value is None or extend):
-                parent_value = getattr(self._play, attr)
+            if self._play and (value is None or extend) and hasattr(self._play, attr):
+                parent_value = getattr(self._play, attr, None)
                 if extend:
                     value = self._extend_value(value, parent_value)
                 else:
                     value = parent_value
-        except KeyError:
+        except KeyError as e:
             pass
 
         return value
@@ -329,6 +353,12 @@ class Block(Base, Become, Conditional, Taggable):
 
         return environment
 
+    def _get_attr_any_errors_fatal(self):
+        '''
+        Override for the 'tags' getattr fetcher, used from Base.
+        '''
+        return self._get_parent_attribute('any_errors_fatal')
+
     def filter_tagged_tasks(self, play_context, all_vars):
         '''
         Creates a new block, with task lists filtered based on the tags contained
@@ -347,7 +377,7 @@ class Block(Base, Become, Conditional, Taggable):
             return tmp_list
 
         def evaluate_block(block):
-            new_block = self.copy()
+            new_block = self.copy(exclude_tasks=True)
             new_block.block  = evaluate_and_append_task(block.block)
             new_block.rescue = evaluate_and_append_task(block.rescue)
             new_block.always = evaluate_and_append_task(block.always)
@@ -357,3 +387,4 @@ class Block(Base, Become, Conditional, Taggable):
 
     def has_tasks(self):
         return len(self.block) > 0 or len(self.rescue) > 0 or len(self.always) > 0
+

lib/ansible/playbook/conditional.py

@@ -22,7 +22,7 @@ __metaclass__ = type
 from jinja2.exceptions import UndefinedError
 
 from ansible.compat.six import text_type
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleUndefinedVariable
 from ansible.playbook.attribute import FieldAttribute
 from ansible.template import Templar
 
@@ -56,7 +56,7 @@ class Conditional:
         False if any of them evaluate as such.
         '''
 
-        # since this is a mixin, it may not have an underlying datastructure
+        # since this is a mix-in, it may not have an underlying datastructure
         # associated with it, so we pull it out now in case we need it for
         # error reporting below
         ds = None
@@ -86,19 +86,25 @@ class Conditional:
         if conditional in all_vars and '-' not in text_type(all_vars[conditional]):
             conditional = all_vars[conditional]
 
-        # make sure the templar is using the variables specifed to this method
+        # make sure the templar is using the variables specified with this method
         templar.set_available_variables(variables=all_vars)
 
-        conditional = templar.template(conditional)
-        if not isinstance(conditional, basestring) or conditional == "":
-            return conditional
+        try:
+            conditional = templar.template(conditional)
+            if not isinstance(conditional, text_type) or conditional == "":
+                return conditional
 
-        # a Jinja2 evaluation that results in something Python can eval!
-        presented = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % conditional
-        conditional = templar.template(presented, fail_on_undefined=False)
-
-        val = conditional.strip()
-        if val == presented:
+            # a Jinja2 evaluation that results in something Python can eval!
+            presented = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % conditional
+            conditional = templar.template(presented)
+            val = conditional.strip()
+            if val == "True":
+                return True
+            elif val == "False":
+                return False
+            else:
+                raise AnsibleError("unable to evaluate conditional: %s" % original)
+        except (AnsibleUndefinedVariable, UndefinedError) as e:
             # the templating failed, meaning most likely a
             # variable was undefined. If we happened to be
             # looking for an undefined variable, return True,
@@ -108,11 +114,5 @@ class Conditional:
             elif "is defined" in original:
                 return False
             else:
-                raise AnsibleError("error while evaluating conditional: %s (%s)" % (original, presented))
-        elif val == "True":
-            return True
-        elif val == "False":
-            return False
-        else:
-            raise AnsibleError("unable to evaluate conditional: %s" % original)
+                raise AnsibleError("error while evaluating conditional (%s): %s" % (original, e))
 

lib/ansible/playbook/helpers.py

@@ -20,9 +20,17 @@ __metaclass__ = type
 
 import os
 
-from ansible.errors import AnsibleParserError
+from ansible import constants as C
+from ansible.compat.six import string_types
+from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleSequence
 
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
 
 def load_list_of_blocks(ds, play, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
     '''
@@ -72,16 +80,18 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h
     from ansible.playbook.block import Block
     from ansible.playbook.handler import Handler
     from ansible.playbook.task import Task
+    from ansible.playbook.task_include import TaskInclude
+    from ansible.template import Templar
 
     assert isinstance(ds, list)
 
     task_list = []
-    for task in ds:
-        assert isinstance(task, dict)
+    for task_ds in ds:
+        assert isinstance(task_ds, dict)
 
-        if 'block' in task:
+        if 'block' in task_ds:
             t = Block.load(
-                task,
+                task_ds,
                 play=play,
                 parent_block=block,
                 role=role,
@@ -90,13 +100,133 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h
                 variable_manager=variable_manager,
                 loader=loader,
             )
+            task_list.append(t)
         else:
-            if use_handlers:
-                t = Handler.load(task, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
-            else:
-                t = Task.load(task, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+            if 'include' in task_ds:
+                t = TaskInclude.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
 
-        task_list.append(t)
+                all_vars = variable_manager.get_vars(loader=loader, play=play, task=t)
+                templar = Templar(loader=loader, variables=all_vars)
+
+                # check to see if this include is static, which can be true if:
+                # 1. the user set the 'static' option to true
+                # 2. one of the appropriate config options was set
+                # 3. the included file name contains no variables, and has no loop
+                is_static = t.static or \
+                            C.DEFAULT_TASK_INCLUDES_STATIC or \
+                            (use_handlers and C.DEFAULT_HANDLER_INCLUDES_STATIC) or \
+                            not templar._contains_vars(t.args.get('_raw_params')) and t.loop is None
+
+                if is_static:
+                    if t.loop is not None:
+                        raise AnsibleParserError("You cannot use 'static' on an include with a loop", obj=task_ds)
+
+                    # FIXME: all of this code is very similar (if not identical) to that in 
+                    #        plugins/strategy/__init__.py, and should be unified to avoid
+                    #        patches only being applied to one or the other location
+                    if task_include:
+                        # handle relative includes by walking up the list of parent include
+                        # tasks and checking the relative result to see if it exists
+                        parent_include = task_include
+                        cumulative_path = None
+                        while parent_include is not None:
+                            parent_include_dir = templar.template(os.path.dirname(parent_include.args.get('_raw_params')))
+                            if cumulative_path is None:
+                                cumulative_path = parent_include_dir
+                            elif not os.path.isabs(cumulative_path):
+                                cumulative_path = os.path.join(parent_include_dir, cumulative_path)
+                            include_target = templar.template(t.args['_raw_params'])
+                            if t._role:
+                                new_basedir = os.path.join(t._role._role_path, 'tasks', cumulative_path)
+                                include_file = loader.path_dwim_relative(new_basedir, 'tasks', include_target)
+                            else:
+                                include_file = loader.path_dwim_relative(loader.get_basedir(), cumulative_path, include_target)
+
+                            if os.path.exists(include_file):
+                                break
+                            else:
+                                parent_include = parent_include._task_include
+                    else:
+                        try:
+                            include_target = templar.template(t.args['_raw_params'])
+                        except AnsibleUndefinedVariable as e:
+                            raise AnsibleParserError(
+                                      "Error when evaluating variable in include name: %s.\n\n" \
+                                      "When using static includes, ensure that any variables used in their names are defined in vars/vars_files\n" \
+                                      "or extra-vars passed in from the command line. Static includes cannot use variables from inventory\n" \
+                                      "sources like group or host vars." % t.args['_raw_params'],
+                                      obj=task_ds,
+                                      suppress_extended_error=True,
+                                  )
+                        if t._role:
+                            if use_handlers:
+                                include_file = loader.path_dwim_relative(t._role._role_path, 'handlers', include_target)
+                            else:
+                                include_file = loader.path_dwim_relative(t._role._role_path, 'tasks', include_target)
+                        else:
+                            include_file = loader.path_dwim(include_target)
+
+                    data = loader.load_from_file(include_file)
+                    if data is None:
+                        return []
+                    elif not isinstance(data, list):
+                        raise AnsibleError("included task files must contain a list of tasks", obj=data)
+
+                    included_blocks = load_list_of_blocks(
+                        data,
+                        play=play,
+                        parent_block=block,
+                        task_include=t,
+                        role=role,
+                        use_handlers=use_handlers,
+                        loader=loader,
+                        variable_manager=variable_manager,
+                    )
+
+                    # Remove the raw params field from the module args, so it won't show up
+                    # later when getting the vars for this task/childen
+                    t.args.pop('_raw_params', None)
+
+                    # pop tags out of the include args, if they were specified there, and assign
+                    # them to the include. If the include already had tags specified, we raise an
+                    # error so that users know not to specify them both ways
+                    tags = t.vars.pop('tags', [])
+                    if isinstance(tags, string_types):
+                        tags = tags.split(',')
+
+                    if len(tags) > 0:
+                        if len(t.tags) > 0:
+                            raise AnsibleParserError(
+                                "Include tasks should not specify tags in more than one way (both via args and directly on the task)." \
+                                " Mixing tag specify styles is prohibited for whole import hierarchy, not only for single import statement",
+                                obj=task_ds,
+                                suppress_extended_error=True,
+                            )
+                        display.deprecated("You should not specify tags in the include parameters. All tags should be specified using the task-level option")
+                    else:
+                        tags = t.tags[:]
+
+                    # now we extend the tags on each of the included blocks
+                    for b in included_blocks:
+                        b.tags = list(set(b.tags).union(tags))
+                    # END FIXME
+
+                    # FIXME: send callback here somehow...
+                    # FIXME: handlers shouldn't need this special handling, but do
+                    #        right now because they don't iterate blocks correctly
+                    if use_handlers:
+                        for b in included_blocks:
+                            task_list.extend(b.block)
+                    else:
+                        task_list.extend(included_blocks)
+                else:
+                    task_list.append(t)
+            elif use_handlers:
+                t = Handler.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+                task_list.append(t)
+            else:
+                t = Task.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+                task_list.append(t)
 
     return task_list
 

lib/ansible/playbook/included_file.py

@@ -49,9 +49,15 @@ class IncludedFile:
         return "%s (%s): %s" % (self._filename, self._args, self._hosts)
 
     @staticmethod
-    def process_include_results(results, tqm, iterator, loader, variable_manager):
+    def process_include_results(results, tqm, iterator, inventory, loader, variable_manager):
         included_files = []
 
+        def get_original_host(host):
+            if host.name in inventory._hosts_cache:
+                return inventory._hosts_cache[host.name]
+            else:
+                return inventory.get_host(host.name)
+
         for res in results:
 
             if res._task.action == 'include':
@@ -67,9 +73,10 @@ class IncludedFile:
                     if 'skipped' in include_result and include_result['skipped'] or 'failed' in include_result:
                         continue
 
-                    original_task = iterator.get_original_task(res._host, res._task)
+                    original_host = get_original_host(res._host)
+                    original_task = iterator.get_original_task(original_host, res._task)
 
-                    task_vars = variable_manager.get_vars(loader=loader, play=iterator._play, host=res._host, task=original_task)
+                    task_vars = variable_manager.get_vars(loader=loader, play=iterator._play, host=original_host, task=original_task)
                     templar = Templar(loader=loader, variables=task_vars)
 
                     include_variables = include_result.get('include_variables', dict())
@@ -77,18 +84,26 @@ class IncludedFile:
                         task_vars['item'] = include_variables['item'] = include_result['item']
 
                     if original_task:
+                        if original_task.static:
+                            continue
+
                         if original_task._task_include:
                             # handle relative includes by walking up the list of parent include
                             # tasks and checking the relative result to see if it exists
                             parent_include = original_task._task_include
+                            cumulative_path = None
                             while parent_include is not None:
                                 parent_include_dir = templar.template(os.path.dirname(parent_include.args.get('_raw_params')))
+                                if cumulative_path is None:
+                                    cumulative_path = parent_include_dir
+                                elif not os.path.isabs(cumulative_path):
+                                    cumulative_path = os.path.join(parent_include_dir, cumulative_path)
                                 include_target = templar.template(include_result['include'])
                                 if original_task._role:
-                                    new_basedir = os.path.join(original_task._role._role_path, 'tasks', parent_include_dir)
+                                    new_basedir = os.path.join(original_task._role._role_path, 'tasks', cumulative_path)
                                     include_file = loader.path_dwim_relative(new_basedir, 'tasks', include_target)
                                 else:
-                                    include_file = loader.path_dwim_relative(loader.get_basedir(), parent_include_dir, include_target)
+                                    include_file = loader.path_dwim_relative(loader.get_basedir(), cumulative_path, include_target)
 
                                 if os.path.exists(include_file):
                                     break
@@ -111,6 +126,6 @@ class IncludedFile:
                     except ValueError:
                         included_files.append(inc_file)
 
-                    inc_file.add_host(res._host)
+                    inc_file.add_host(original_host)
 
         return included_files

lib/ansible/playbook/play.py

@@ -64,7 +64,8 @@ class Play(Base, Taggable, Become):
 
     # Connection
     _gather_facts        = FieldAttribute(isa='bool', default=None, always_post_validate=True)
-    _hosts               = FieldAttribute(isa='list', default=[], required=True, listof=string_types, always_post_validate=True)
+    _gather_subset       = FieldAttribute(isa='list', default=None, always_post_validate=True)
+    _hosts               = FieldAttribute(isa='list', required=True, listof=string_types, always_post_validate=True)
     _name                = FieldAttribute(isa='string', default='', always_post_validate=True)
 
     # Variable Attributes
@@ -105,6 +106,11 @@ class Play(Base, Taggable, Become):
 
     @staticmethod
     def load(data, variable_manager=None, loader=None):
+        if ('name' not in data or data['name'] is None) and 'hosts' in data:
+            if isinstance(data['hosts'], list):
+                data['name'] = ','.join(data['hosts'])
+            else:
+                 data['name'] = data['hosts']
         p = Play()
         return p.load_data(data, variable_manager=variable_manager, loader=loader)
 

lib/ansible/playbook/play_context.py

@@ -50,7 +50,7 @@ except ImportError:
 
 MAGIC_VARIABLE_MAPPING = dict(
    connection       = ('ansible_connection',),
-   connection_args  = ('ansible_connection_args',),
+   docker_extra_args  = ('ansible_docker_extra_args',),
    remote_addr      = ('ansible_ssh_host', 'ansible_host'),
    remote_user      = ('ansible_ssh_user', 'ansible_user'),
    port             = ('ansible_ssh_port', 'ansible_port'),
@@ -79,6 +79,7 @@ MAGIC_VARIABLE_MAPPING = dict(
    su_pass          = ('ansible_su_password', 'ansible_su_pass'),
    su_exe           = ('ansible_su_exe',),
    su_flags         = ('ansible_su_flags',),
+   executable       = ('ansible_shell_executable',),
 )
 
 SU_PROMPT_LOCALIZATIONS = [
@@ -121,11 +122,25 @@ TASK_ATTRIBUTE_OVERRIDES = (
     'become_pass',
     'become_method',
     'connection',
+    'docker_extra_args',
     'delegate_to',
     'no_log',
     'remote_user',
 )
 
+RESET_VARS = (
+    'ansible_connection',
+    'ansible_docker_extra_args',
+    'ansible_ssh_host',
+    'ansible_ssh_pass',
+    'ansible_ssh_port',
+    'ansible_ssh_user',
+    'ansible_ssh_private_key_file',
+    'ansible_ssh_pipelining',
+    'ansible_user',
+    'ansible_host',
+    'ansible_port',
+)
 
 class PlayContext(Base):
 
@@ -137,7 +152,7 @@ class PlayContext(Base):
 
     # connection fields, some are inherited from Base:
     # (connection, port, remote_user, environment, no_log)
-    _connection_args  = FieldAttribute(isa='string')
+    _docker_extra_args  = FieldAttribute(isa='string')
     _remote_addr      = FieldAttribute(isa='string')
     _password         = FieldAttribute(isa='string')
     _private_key_file = FieldAttribute(isa='string', default=C.DEFAULT_PRIVATE_KEY_FILE)
@@ -153,6 +168,7 @@ class PlayContext(Base):
     _accelerate       = FieldAttribute(isa='bool', default=False)
     _accelerate_ipv6  = FieldAttribute(isa='bool', default=False, always_post_validate=True)
     _accelerate_port  = FieldAttribute(isa='int', default=C.ACCELERATE_PORT, always_post_validate=True)
+    _executable       = FieldAttribute(isa='string', default=C.DEFAULT_EXECUTABLE)
 
     # privilege escalation fields
     _become           = FieldAttribute(isa='bool')
@@ -245,34 +261,23 @@ class PlayContext(Base):
         if options.connection:
             self.connection = options.connection
 
-        if hasattr(options, 'connection_args') and options.connection_args:
-            self.connection_args = options.connection_args
-
-        self.remote_user = options.remote_user
-        self.private_key_file = options.private_key_file
-        self.ssh_common_args = options.ssh_common_args
-        self.sftp_extra_args = options.sftp_extra_args
-        self.scp_extra_args = options.scp_extra_args
-        self.ssh_extra_args = options.ssh_extra_args
-
         # privilege escalation
         self.become        = options.become
         self.become_method = options.become_method
         self.become_user   = options.become_user
 
+        self.check_mode = boolean(options.check)
+
+        # get ssh options FIXME: make these common to all connections
+        for flag in ['ssh_common_args', 'docker_extra_args', 'sftp_extra_args', 'scp_extra_args', 'ssh_extra_args']:
+            setattr(self, flag, getattr(options,flag, ''))
+
         # general flags (should we move out?)
-        if options.verbosity:
-            self.verbosity  = options.verbosity
-        if options.check:
-            self.check_mode = boolean(options.check)
-        if hasattr(options, 'force_handlers') and options.force_handlers:
-            self.force_handlers = boolean(options.force_handlers)
-        if hasattr(options, 'step') and options.step:
-            self.step = boolean(options.step)
-        if hasattr(options, 'start_at_task') and options.start_at_task:
-            self.start_at_task = to_unicode(options.start_at_task)
-        if hasattr(options, 'diff') and options.diff:
-            self.diff = boolean(options.diff)
+        for flag in ['connection','remote_user', 'private_key_file', 'verbosity', 'force_handlers', 'step', 'start_at_task', 'diff']:
+            attribute = getattr(options, flag, False)
+            if attribute:
+                setattr(self, flag, attribute)
+
         if hasattr(options, 'timeout') and options.timeout:
             self.timeout = int(options.timeout)
 
@@ -359,24 +364,54 @@ class PlayContext(Base):
         else:
             delegated_vars = dict()
 
+            # setup shell
+            for exe_var in MAGIC_VARIABLE_MAPPING.get('executable'):
+                if exe_var in variables:
+                    setattr(new_info, 'executable', variables.get(exe_var))
+
+        attrs_considered = []
         for (attr, variable_names) in iteritems(MAGIC_VARIABLE_MAPPING):
             for variable_name in variable_names:
-                if isinstance(delegated_vars, dict) and variable_name in delegated_vars:
-                    setattr(new_info, attr, delegated_vars[variable_name])
+                if attr in attrs_considered:
+                    continue
+                # if delegation task ONLY use delegated host vars, avoid delegated FOR host vars
+                if task.delegate_to is not None:
+                    if isinstance(delegated_vars, dict) and variable_name in delegated_vars:
+                        setattr(new_info, attr, delegated_vars[variable_name])
+                        attrs_considered.append(attr)
                 elif variable_name in variables:
                     setattr(new_info, attr, variables[variable_name])
+                    attrs_considered.append(attr)
+                # no else, as no other vars should be considered
 
-        # make sure we get port defaults if needed
-        if new_info.port is None and C.DEFAULT_REMOTE_PORT is not None:
-            new_info.port = int(C.DEFAULT_REMOTE_PORT)
-
-        # become legacy updates
+        # become legacy updates -- from commandline
         if not new_info.become_pass:
             if new_info.become_method == 'sudo' and new_info.sudo_pass:
                setattr(new_info, 'become_pass', new_info.sudo_pass)
             elif new_info.become_method == 'su' and new_info.su_pass:
                setattr(new_info, 'become_pass', new_info.su_pass)
 
+        # become legacy updates -- from inventory file (inventory overrides
+        # commandline)
+        for become_pass_name in MAGIC_VARIABLE_MAPPING.get('become_pass'):
+            if become_pass_name in variables:
+                break
+        else:  # This is a for-else
+            if new_info.become_method == 'sudo':
+                for sudo_pass_name in MAGIC_VARIABLE_MAPPING.get('sudo_pass'):
+                    if sudo_pass_name in variables:
+                        setattr(new_info, 'become_pass', variables[sudo_pass_name])
+                        break
+            if new_info.become_method == 'sudo':
+                for su_pass_name in MAGIC_VARIABLE_MAPPING.get('su_pass'):
+                    if su_pass_name in variables:
+                        setattr(new_info, 'become_pass', variables[su_pass_name])
+                        break
+
+        # make sure we get port defaults if needed
+        if new_info.port is None and C.DEFAULT_REMOTE_PORT is not None:
+            new_info.port = int(C.DEFAULT_REMOTE_PORT)
+
         # special overrides for the connection setting
         if len(delegated_vars) > 0:
             # in the event that we were using local before make sure to reset the
@@ -397,6 +432,13 @@ class PlayContext(Base):
         if new_info.no_log is None:
             new_info.no_log = C.DEFAULT_NO_LOG
 
+        # set become defaults if not previouslly set
+        task.set_become_defaults(new_info.become, new_info.become_method, new_info.become_user)
+
+        # have always_run override check mode
+        if task.always_run:
+            new_info.check_mode = False
+
         return new_info
 
     def make_become_cmd(self, cmd, executable=None):
@@ -407,7 +449,7 @@ class PlayContext(Base):
         self.prompt = None
 
         if executable is None:
-            executable = C.DEFAULT_EXECUTABLE
+            executable = self.executable
 
         if self.become:
 
@@ -432,8 +474,10 @@ class PlayContext(Base):
 
             if self.become_method == 'sudo':
                 # If we have a password, we run sudo with a randomly-generated
-                # prompt set using -p. Otherwise we run it with -n, which makes
+                # prompt set using -p. Otherwise we run it with default -n, which makes
                 # it fail if it would have prompted for a password.
+                # Cannot rely on -n as it can be removed from defaults, which should be
+                # done for older versions of sudo that do not support the option.
                 #
                 # Passing a quoted compound command to sudo (or sudo -s)
                 # directly doesn't work, so we shellquote it with pipes.quote()
@@ -449,12 +493,13 @@ class PlayContext(Base):
 
             elif self.become_method == 'su':
 
+                # passing code ref to examine prompt as simple string comparisson isn't good enough with su
                 def detect_su_prompt(data):
                     SU_PROMPT_LOCALIZATIONS_RE = re.compile("|".join(['(\w+\'s )?' + x + ' ?: ?' for x in SU_PROMPT_LOCALIZATIONS]), flags=re.IGNORECASE)
                     return bool(SU_PROMPT_LOCALIZATIONS_RE.match(data))
-
                 prompt = detect_su_prompt
-                becomecmd = '%s %s %s -c "%s -c %s"' % (exe, flags, self.become_user, executable, success_cmd)
+
+                becomecmd = '%s %s %s -c %s' % (exe, flags, self.become_user, pipes.quote('%s -c %s' % (executable, success_cmd)))
 
             elif self.become_method == 'pbrun':
 
@@ -468,7 +513,7 @@ class PlayContext(Base):
 
             elif self.become_method == 'runas':
                 raise AnsibleError("'runas' is not yet implemented")
-                #TODO: figure out prompt
+                #FIXME: figure out prompt
                 # this is not for use with winrm plugin but if they ever get ssh native on windoez
                 becomecmd = '%s %s /user:%s "%s"' % (exe, flags, self.become_user, success_cmd)
 
@@ -483,6 +528,7 @@ class PlayContext(Base):
                 if self.become_user:
                     flags += ' -u %s ' % self.become_user
 
+                #FIXME: make shell independant
                 becomecmd = '%s %s echo %s && %s %s env ANSIBLE=true %s' % (exe, flags, success_key, exe, flags, cmd)
 
             else:
@@ -491,7 +537,7 @@ class PlayContext(Base):
             if self.become_pass:
                 self.prompt = prompt
             self.success_key = success_key
-            return ('%s -c %s' % (executable, pipes.quote(becomecmd)))
+            return becomecmd
 
         return cmd
 
@@ -501,10 +547,14 @@ class PlayContext(Base):
         In case users need to access from the play, this is a legacy from runner.
         '''
 
-        # TODO: should we be setting the more generic values here rather than
-        #       the more specific _ssh_ ones?
-        for special_var in  ['ansible_connection', 'ansible_ssh_host', 'ansible_ssh_pass', 'ansible_ssh_port', 'ansible_ssh_user', 'ansible_ssh_private_key_file', 'ansible_ssh_pipelining']:
-            if special_var not in variables:
-                for prop, varnames in MAGIC_VARIABLE_MAPPING.items():
-                    if special_var in varnames:
-                        variables[special_var] = getattr(self, prop)
+        for prop, var_list in MAGIC_VARIABLE_MAPPING.items():
+            try:
+                if 'become' in prop:
+                    continue
+
+                var_val = getattr(self, prop)
+                for var_opt in var_list:
+                    if var_opt not in variables and var_val is not None:
+                        variables[var_opt] = var_val
+            except AttributeError:
+                continue

lib/ansible/playbook/playbook_include.py

@@ -22,7 +22,7 @@ __metaclass__ = type
 import os
 
 from ansible.compat.six import iteritems
-from ansible.errors import AnsibleParserError
+from ansible.errors import AnsibleParserError, AnsibleError
 from ansible.parsing.splitter import split_args, parse_kv
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
 from ansible.playbook.attribute import FieldAttribute
@@ -60,8 +60,15 @@ class PlaybookInclude(Base, Conditional, Taggable):
             all_vars.update(variable_manager.get_vars(loader=loader))
 
         templar = Templar(loader=loader, variables=all_vars)
-        if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
-            return None
+
+        try:
+            forward_conditional = False
+            if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
+                return None
+        except AnsibleError:
+            # conditional evaluation raised an error, so we set a flag to indicate
+            # we need to forward the conditionals on to the included play(s)
+            forward_conditional = True
 
         # then we use the object to load a Playbook
         pb = Playbook(loader=loader)
@@ -85,6 +92,13 @@ class PlaybookInclude(Base, Conditional, Taggable):
             if entry._included_path is None:
                 entry._included_path = os.path.dirname(file_name)
 
+            # Check to see if we need to forward the conditionals on to the included
+            # plays. If so, we can take a shortcut here and simply prepend them to
+            # those attached to each block (if any)
+            if forward_conditional:
+                for task_block in entry.tasks:
+                    task_block.when = self.when[:] + task_block.when
+
         return pb
 
     def preprocess_data(self, ds):

lib/ansible/playbook/role/__init__.py

@@ -43,7 +43,10 @@ __all__ = ['Role', 'hash_params']
 #       strategies (ansible/plugins/strategy/__init__.py)
 def hash_params(params):
     if not isinstance(params, dict):
-        return params
+        if isinstance(params, list):
+            return frozenset(params)
+        else:
+            return params
     else:
         s = set()
         for k,v in iteritems(params):
@@ -61,6 +64,7 @@ def hash_params(params):
 class Role(Base, Become, Conditional, Taggable):
 
     _delegate_to = FieldAttribute(isa='string')
+    _delegate_facts = FieldAttribute(isa='bool', default=False)
 
     def __init__(self, play=None):
         self._role_name        = None
@@ -149,7 +153,7 @@ class Role(Base, Become, Conditional, Taggable):
         current_when = getattr(self, 'when')[:]
         current_when.extend(role_include.when)
         setattr(self, 'when', current_when)
-        
+
         current_tags = getattr(self, 'tags')[:]
         current_tags.extend(role_include.tags)
         setattr(self, 'tags', current_tags)
@@ -172,16 +176,16 @@ class Role(Base, Become, Conditional, Taggable):
         task_data = self._load_role_yaml('tasks')
         if task_data:
             try:
-                self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader)
-            except:
+                self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager)
+            except AssertionError:
                 raise AnsibleParserError("The tasks/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=task_data)
 
         handler_data = self._load_role_yaml('handlers')
         if handler_data:
             try:
-                self._handler_blocks = load_list_of_blocks(handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader)
-            except:
-                raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=task_data)
+                self._handler_blocks = load_list_of_blocks(handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader, variable_manager=self._variable_manager)
+            except AssertionError:
+                raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=handler_data)
 
         # vars and default vars are regular dictionaries
         self._role_vars  = self._load_role_yaml('vars')
@@ -248,31 +252,41 @@ class Role(Base, Become, Conditional, Taggable):
     def get_parents(self):
         return self._parents
 
-    def get_default_vars(self):
+    def get_default_vars(self, dep_chain=[]):
         default_vars = dict()
         for dep in self.get_all_dependencies():
             default_vars = combine_vars(default_vars, dep.get_default_vars())
+        if dep_chain:
+            for parent in dep_chain:
+                default_vars = combine_vars(default_vars, parent._default_vars)
         default_vars = combine_vars(default_vars, self._default_vars)
         return default_vars
 
-    def get_inherited_vars(self, dep_chain=[], include_params=True):
+    def get_inherited_vars(self, dep_chain=[]):
         inherited_vars = dict()
 
-        for parent in dep_chain:
-            inherited_vars = combine_vars(inherited_vars, parent._role_vars)
-            if include_params:
-                inherited_vars = combine_vars(inherited_vars, parent._role_params)
+        if dep_chain:
+            for parent in dep_chain:
+                inherited_vars = combine_vars(inherited_vars, parent._role_vars)
         return inherited_vars
 
+    def get_role_params(self, dep_chain=[]):
+        params = {}
+        if dep_chain:
+            for parent in dep_chain:
+                params = combine_vars(params, parent._role_params)
+        params = combine_vars(params, self._role_params)
+        return params
+
     def get_vars(self, dep_chain=[], include_params=True):
-        all_vars = self.get_inherited_vars(dep_chain, include_params=include_params)
+        all_vars = self.get_inherited_vars(dep_chain)
 
         for dep in self.get_all_dependencies():
             all_vars = combine_vars(all_vars, dep.get_vars(include_params=include_params))
 
         all_vars = combine_vars(all_vars, self._role_vars)
         if include_params:
-            all_vars = combine_vars(all_vars, self._role_params)
+            all_vars = combine_vars(all_vars, self.get_role_params(dep_chain=dep_chain))
 
         return all_vars
 
@@ -313,7 +327,7 @@ class Role(Base, Become, Conditional, Taggable):
 
         return host.name in self._completed and not self._metadata.allow_duplicates
 
-    def compile(self, play, dep_chain=[]):
+    def compile(self, play, dep_chain=None):
         '''
         Returns the task list for this role, which is created by first
         recursively compiling the tasks for all direct dependencies, and
@@ -327,18 +341,20 @@ class Role(Base, Become, Conditional, Taggable):
         block_list = []
 
         # update the dependency chain here
+        if dep_chain is None:
+            dep_chain = []
         new_dep_chain = dep_chain + [self]
 
         deps = self.get_direct_dependencies()
         for dep in deps:
             dep_blocks = dep.compile(play=play, dep_chain=new_dep_chain)
-            for dep_block in dep_blocks:
-                new_dep_block = dep_block.copy()
-                new_dep_block._dep_chain = new_dep_chain
-                new_dep_block._play = play
-                block_list.append(new_dep_block)
+            block_list.extend(dep_blocks)
 
-        block_list.extend(self._task_blocks)
+        for task_block in self._task_blocks:
+            new_task_block = task_block.copy()
+            new_task_block._dep_chain = new_dep_chain
+            new_task_block._play = play
+            block_list.append(new_task_block)
 
         return block_list
 

lib/ansible/playbook/role/definition.py

@@ -135,46 +135,44 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
         append it to the default role path
         '''
 
-        role_path = unfrackpath(role_name)
+        # we always start the search for roles in the base directory of the playbook
+        role_search_paths = [
+            os.path.join(self._loader.get_basedir(), u'roles'),
+            self._loader.get_basedir(),
+        ]
 
+        # also search in the configured roles path
+        if C.DEFAULT_ROLES_PATH:
+            configured_paths = C.DEFAULT_ROLES_PATH.split(os.pathsep)
+            role_search_paths.extend(configured_paths)
+
+        # finally, append the roles basedir, if it was set, so we can
+        # search relative to that directory for dependent roles
+        if self._role_basedir:
+            role_search_paths.append(self._role_basedir)
+
+        # create a templar class to template the dependency names, in
+        # case they contain variables
+        if self._variable_manager is not None:
+            all_vars = self._variable_manager.get_vars(loader=self._loader, play=self._play)
+        else:
+            all_vars = dict()
+
+        templar = Templar(loader=self._loader, variables=all_vars)
+        role_name = templar.template(role_name)
+
+        # now iterate through the possible paths and return the first one we find
+        for path in role_search_paths:
+            path = templar.template(path)
+            role_path = unfrackpath(os.path.join(path, role_name))
+            if self._loader.path_exists(role_path):
+                return (role_name, role_path)
+
+        # if not found elsewhere try to extract path from name
+        role_path = unfrackpath(role_name)
         if self._loader.path_exists(role_path):
             role_name = os.path.basename(role_name)
             return (role_name, role_path)
-        else:
-            # we always start the search for roles in the base directory of the playbook
-            role_search_paths = [
-                os.path.join(self._loader.get_basedir(), u'roles'),
-                u'./roles',
-                self._loader.get_basedir(),
-                u'./'
-            ]
-
-            # also search in the configured roles path
-            if C.DEFAULT_ROLES_PATH:
-                configured_paths = C.DEFAULT_ROLES_PATH.split(os.pathsep)
-                role_search_paths.extend(configured_paths)
-
-            # finally, append the roles basedir, if it was set, so we can
-            # search relative to that directory for dependent roles
-            if self._role_basedir:
-                role_search_paths.append(self._role_basedir)
-
-            # create a templar class to template the dependency names, in
-            # case they contain variables
-            if self._variable_manager is not None:
-                all_vars = self._variable_manager.get_vars(loader=self._loader, play=self._play)
-            else:
-                all_vars = dict()
-
-            templar = Templar(loader=self._loader, variables=all_vars)
-            role_name = templar.template(role_name)
-
-            # now iterate through the possible paths and return the first one we find
-            for path in role_search_paths:
-                path = templar.template(path)
-                role_path = unfrackpath(os.path.join(path, role_name))
-                if self._loader.path_exists(role_path):
-                    return (role_name, role_path)
 
         raise AnsibleError("the role '%s' was not found in %s" % (role_name, ":".join(role_search_paths)), obj=self._ds)
 
@@ -190,7 +188,12 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
         for (key, value) in iteritems(ds):
             # use the list of FieldAttribute values to determine what is and is not
             # an extra parameter for this role (or sub-class of this role)
-            if key not in base_attribute_names:
+            # FIXME: hard-coded list of exception key names here corresponds to the
+            #        connection fields in the Base class. There may need to be some
+            #        other mechanism where we exclude certain kinds of field attributes,
+            #        or make this list more automatic in some way so we don't have to
+            #        remember to update it manually.
+            if key not in base_attribute_names or key in ('connection', 'port', 'remote_user'):
                 # this key does not match a field attribute, so it must be a role param
                 role_params[key] = value
             else:

lib/ansible/playbook/role/include.py

@@ -40,7 +40,8 @@ class RoleInclude(RoleDefinition):
     is included for execution in a play.
     """
 
-    _delegate_to          = FieldAttribute(isa='string')
+    _delegate_to    = FieldAttribute(isa='string')
+    _delegate_facts = FieldAttribute(isa='bool', default=False)
 
     def __init__(self, play=None, role_basedir=None, variable_manager=None, loader=None):
         super(RoleInclude, self).__init__(play=play, role_basedir=role_basedir, variable_manager=variable_manager, loader=loader)

lib/ansible/playbook/role/requirement.py

@@ -190,6 +190,17 @@ class RoleRequirement(RoleDefinition):
         if rc != 0:
             raise AnsibleError ("- command %s failed in directory %s (rc=%s)" % (' '.join(clone_cmd), tempdir, rc))
 
+        if scm == 'git' and version:
+            checkout_cmd = [scm, 'checkout', version]
+            with open('/dev/null', 'w') as devnull:
+                try:
+                    popen = subprocess.Popen(checkout_cmd, cwd=os.path.join(tempdir, name), stdout=devnull, stderr=devnull)
+                except (IOError, OSError):
+                    raise AnsibleError("error executing: %s" % " ".join(checkout_cmd))
+                rc = popen.wait()
+            if rc != 0:
+                raise AnsibleError("- command %s failed in directory %s (rc=%s)" % (' '.join(checkout_cmd), tempdir, rc))
+
         temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.tar')
         if scm == 'hg':
             archive_cmd = ['hg', 'archive', '--prefix', "%s/" % name]

lib/ansible/playbook/taggable.py

@@ -38,7 +38,11 @@ class Taggable:
         if isinstance(ds, list):
             return ds
         elif isinstance(ds, basestring):
-            return [ ds ]
+            value = ds.split(',')
+            if isinstance(value, list):
+                return [ x.strip() for x in value ]
+            else:
+                return [ ds ]
         else:
             raise AnsibleError('tags must be specified as a list', obj=ds)
 

lib/ansible/playbook/task.py

@@ -21,13 +21,12 @@ __metaclass__ = type
 
 from ansible.compat.six import iteritems, string_types
 
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleParserError
 
 from ansible.parsing.mod_args import ModuleArgsParser
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping, AnsibleUnicode
 
 from ansible.plugins import lookup_loader
-
 from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.become import Become
@@ -36,6 +35,8 @@ from ansible.playbook.conditional import Conditional
 from ansible.playbook.role import Role
 from ansible.playbook.taggable import Taggable
 
+from ansible.utils.unicode import to_str
+
 try:
     from __main__ import display
 except ImportError:
@@ -69,10 +70,11 @@ class Task(Base, Conditional, Taggable, Become):
 
     _any_errors_fatal     = FieldAttribute(isa='bool')
     _async                = FieldAttribute(isa='int', default=0)
-    _changed_when         = FieldAttribute(isa='string')
+    _changed_when         = FieldAttribute(isa='list', default=[])
     _delay                = FieldAttribute(isa='int', default=5)
     _delegate_to          = FieldAttribute(isa='string')
-    _failed_when          = FieldAttribute(isa='string')
+    _delegate_facts       = FieldAttribute(isa='bool', default=False)
+    _failed_when          = FieldAttribute(isa='list', default=[])
     _first_available_file = FieldAttribute(isa='list')
     _loop                 = FieldAttribute(isa='string', private=True)
     _loop_args            = FieldAttribute(isa='list', private=True)
@@ -81,7 +83,7 @@ class Task(Base, Conditional, Taggable, Become):
     _poll                 = FieldAttribute(isa='int')
     _register             = FieldAttribute(isa='string')
     _retries              = FieldAttribute(isa='int', default=3)
-    _until                = FieldAttribute(isa='list')
+    _until                = FieldAttribute(isa='list', default=[])
 
     def __init__(self, block=None, role=None, task_include=None):
         ''' constructors a task, without the Task.load classmethod, it will be pretty blank '''
@@ -106,11 +108,10 @@ class Task(Base, Conditional, Taggable, Become):
         elif self.name:
             return self.name
         else:
-            flattened_args = self._merge_kv(self.args)
             if self._role:
-                return "%s : %s %s" % (self._role.get_name(), self.action, flattened_args)
+                return "%s : %s" % (self._role.get_name(), self.action)
             else:
-                return "%s %s" % (self.action, flattened_args)
+                return "%s" % (self.action,)
 
     def _merge_kv(self, ds):
         if ds is None:
@@ -133,7 +134,10 @@ class Task(Base, Conditional, Taggable, Become):
 
     def __repr__(self):
         ''' returns a human readable representation of the task '''
-        return "TASK: %s" % self.get_name()
+        if self.get_name() == 'meta':
+            return "TASK: meta (%s)" % self.args['_raw_params']
+        else:
+            return "TASK: %s" % self.get_name()
 
     def _preprocess_loop(self, ds, new_ds, k, v):
         ''' take a lookup plugin name and store it correctly '''
@@ -165,7 +169,10 @@ class Task(Base, Conditional, Taggable, Become):
         # and the delegate_to value from the various possible forms
         # supported as legacy
         args_parser = ModuleArgsParser(task_ds=ds)
-        (action, args, delegate_to) = args_parser.parse()
+        try:
+            (action, args, delegate_to) = args_parser.parse()
+        except AnsibleParserError as e:
+            raise AnsibleParserError(to_str(e), obj=ds)
 
         # the command/shell/script modules used to support the `cmd` arg,
         # which corresponds to what we now call _raw_params, so move that
@@ -213,14 +220,6 @@ class Task(Base, Conditional, Taggable, Become):
 
         return super(Task, self).preprocess_data(new_ds)
 
-    def _load_any_errors_fatal(self, attr, value):
-        '''
-        Exists only to show a deprecation warning, as this attribute is not valid
-        at the task level.
-        '''
-        display.deprecated("Setting any_errors_fatal on a task is no longer supported. This should be set at the play level only")
-        return None
-
     def post_validate(self, templar):
         '''
         Override of base class post_validate, to also do final validation on
@@ -234,6 +233,13 @@ class Task(Base, Conditional, Taggable, Become):
 
         super(Task, self).post_validate(templar)
 
+    def _post_validate_register(self, attr, value, templar):
+        '''
+        Override post validation for the register args field, which is not
+        supposed to be templated
+        '''
+        return value
+
     def _post_validate_loop_args(self, attr, value, templar):
         '''
         Override post validation for the loop args field, which is templated
@@ -249,13 +255,44 @@ class Task(Base, Conditional, Taggable, Become):
         if value is None:
             return dict()
 
-        for env_item in value:
-            if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys():
-                display.deprecated("Using bare variables for environment is deprecated."
-                        " Update your playbooks so that the environment value uses the full variable syntax ('{{foo}}')")
-                break
+        elif isinstance(value, list):
+            if  len(value) == 1:
+                return templar.template(value[0], convert_bare=True)
+            else:
+                env = []
+                for env_item in value:
+                    if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys():
+                        env[env_item] =  templar.template(env_item, convert_bare=True)
+        elif isinstance(value, dict):
+            env = dict()
+            for env_item in value:
+                if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys():
+                    env[env_item] =  templar.template(value[env_item], convert_bare=True)
+
+        # at this point it should be a simple string
         return templar.template(value, convert_bare=True)
 
+    def _post_validate_changed_when(self, attr, value, templar):
+        '''
+        changed_when is evaluated after the execution of the task is complete,
+        and should not be templated during the regular post_validate step.
+        '''
+        return value
+
+    def _post_validate_failed_when(self, attr, value, templar):
+        '''
+        failed_when is evaluated after the execution of the task is complete,
+        and should not be templated during the regular post_validate step.
+        '''
+        return value
+
+    def _post_validate_until(self, attr, value, templar):
+        '''
+        until is evaluated after the execution of the task is complete,
+        and should not be templated during the regular post_validate step.
+        '''
+        return value
+
     def get_vars(self):
         all_vars = dict()
         if self._block:
@@ -398,3 +435,10 @@ class Task(Base, Conditional, Taggable, Become):
         if parent_environment is not None:
             environment = self._extend_value(environment, parent_environment)
         return environment
+
+    def _get_attr_any_errors_fatal(self):
+        '''
+        Override for the 'tags' getattr fetcher, used from Base.
+        '''
+        return self._get_parent_attribute('any_errors_fatal')
+

lib/ansible/playbook/task_include.py

@@ -0,0 +1,72 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.playbook.attribute import FieldAttribute
+from ansible.playbook.task import Task
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+__all__ = ['TaskInclude']
+
+
+class TaskInclude(Task):
+
+    """
+    A task include is derived from a regular task to handle the special
+    circumstances related to the `- include: ...` task.
+    """
+
+    # =================================================================================
+    # ATTRIBUTES
+
+    _static = FieldAttribute(isa='bool', default=False)
+
+    @staticmethod
+    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
+        t = TaskInclude(block=block, role=role, task_include=task_include)
+        return t.load_data(data, variable_manager=variable_manager, loader=loader)
+
+    def get_vars(self):
+        '''
+        We override the parent Task() classes get_vars here because
+        we need to include the args of the include into the vars as
+        they are params to the included tasks.
+        '''
+        all_vars = dict()
+        if self._block:
+            all_vars.update(self._block.get_vars())
+        if self._task_include:
+            all_vars.update(self._task_include.get_vars())
+
+        all_vars.update(self.vars)
+        all_vars.update(self.args)
+
+        if 'tags' in all_vars:
+            del all_vars['tags']
+        if 'when' in all_vars:
+            del all_vars['when']
+
+        return all_vars
+

lib/ansible/plugins/__init__.py

@@ -316,6 +316,7 @@ class PluginLoader:
     def get(self, name, *args, **kwargs):
         ''' instantiates a plugin of the given name using arguments '''
 
+        class_only = kwargs.pop('class_only', False)
         if name in self.aliases:
             name = self.aliases[name]
         path = self.find_plugin(name)
@@ -325,40 +326,60 @@ class PluginLoader:
         if path not in self._module_cache:
             self._module_cache[path] = self._load_module_source('.'.join([self.package, name]), path)
 
-        if kwargs.get('class_only', False):
-            obj = getattr(self._module_cache[path], self.class_name)
-        else:
-            obj = getattr(self._module_cache[path], self.class_name)(*args, **kwargs)
-            if self.base_class and self.base_class not in [base.__name__ for base in obj.__class__.__bases__]:
+        obj = getattr(self._module_cache[path], self.class_name)
+        if self.base_class:
+            # The import path is hardcoded and should be the right place,
+            # so we are not expecting an ImportError.
+            module = __import__(self.package, fromlist=[self.base_class])
+            # Check whether this obj has the required base class.
+            try:
+                plugin_class = getattr(module, self.base_class)
+            except AttributeError:
                 return None
+            if not issubclass(obj, plugin_class):
+                return None
+
+        if not class_only:
+            obj = obj(*args, **kwargs)
 
         return obj
 
     def all(self, *args, **kwargs):
         ''' instantiates all plugins with the same arguments '''
 
+        class_only = kwargs.pop('class_only', False)
+        all_matches = []
+ 
         for i in self._get_paths():
-            matches = glob.glob(os.path.join(i, "*.py"))
-            matches.sort()
-            for path in matches:
-                name, _ = os.path.splitext(path)
-                if '__init__' in name:
-                    continue
+            all_matches.extend(glob.glob(os.path.join(i, "*.py")))
 
-                if path not in self._module_cache:
-                    self._module_cache[path] = self._load_module_source(name, path)
+        for path in sorted(all_matches, key=lambda match: os.path.basename(match)):
+            name, _ = os.path.splitext(path)
+            if '__init__' in name:
+                continue
 
-                if kwargs.get('class_only', False):
-                    obj = getattr(self._module_cache[path], self.class_name)
-                else:
-                    obj = getattr(self._module_cache[path], self.class_name)(*args, **kwargs)
+            if path not in self._module_cache:
+                self._module_cache[path] = self._load_module_source(name, path)
 
-                    if self.base_class and self.base_class not in [base.__name__ for base in obj.__class__.__bases__]:
-                        continue
+            obj = getattr(self._module_cache[path], self.class_name)
+            if self.base_class:
+                # The import path is hardcoded and should be the right place,
+                # so we are not expecting an ImportError.
+                module = __import__(self.package, fromlist=[self.base_class])
+                # Check whether this obj has the required base class.
+                try:
+                   plugin_class = getattr(module, self.base_class)
+                except AttributeError:
+                   continue
+                if not issubclass(obj, plugin_class):
+                   continue
 
-                # set extra info on the module, in case we want it later
-                setattr(obj, '_original_path', path)
-                yield obj
+            if not class_only:
+                obj = obj(*args, **kwargs)
+
+            # set extra info on the module, in case we want it later
+            setattr(obj, '_original_path', path)
+            yield obj
 
 action_loader = PluginLoader(
     'ActionModule',
@@ -444,7 +465,7 @@ fragment_loader = PluginLoader(
 strategy_loader = PluginLoader(
     'StrategyModule',
     'ansible.plugins.strategy',
-    None,
+    C.DEFAULT_STRATEGY_PLUGIN_PATH,
     'strategy_plugins',
     required_base_class='StrategyBase',
 )

lib/ansible/plugins/action/__init__.py

@@ -24,6 +24,7 @@ import json
 import os
 import pipes
 import random
+import re
 import stat
 import tempfile
 import time
@@ -119,7 +120,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             module_path = self._shared_loader_obj.module_loader.find_plugin(module_name, mod_type)
             if module_path:
                 break
-        else:
+        else:  # This is a for-else: http://bit.ly/1ElPkyg
             # Use Windows version of ping module to check module paths when
             # using a connection that supports .ps1 suffixes. We check specifically
             # for win_ping here, otherwise the code would look for ping.ps1
@@ -151,14 +152,19 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             if not isinstance(environments, list):
                 environments = [ environments ]
 
+            # the environments as inherited need to be reversed, to make
+            # sure we merge in the parent's values first so those in the
+            # block then task 'win' in precedence
+            environments.reverse()
             for environment in environments:
                 if environment is None:
                     continue
-                if not isinstance(environment, dict):
-                    raise AnsibleError("environment must be a dictionary, received %s (%s)" % (environment, type(environment)))
+                temp_environment = self._templar.template(environment)
+                if not isinstance(temp_environment, dict):
+                    raise AnsibleError("environment must be a dictionary, received %s (%s)" % (temp_environment, type(temp_environment)))
                 # very deliberately using update here instead of combine_vars, as
                 # these environment settings should not need to merge sub-dicts
-                final_environment.update(environment)
+                final_environment.update(temp_environment)
 
         final_environment = self._templar.template(final_environment)
         return self._connection._shell.env_prefix(**final_environment)
@@ -186,7 +192,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             return True
         return False
 
-    def _make_tmp_path(self):
+    def _make_tmp_path(self, remote_user):
         '''
         Create and return a temporary path on a remote box.
         '''
@@ -194,17 +200,13 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))
         use_system_tmp = False
 
-        if self._play_context.become and self._play_context.become_user != 'root':
+        if self._play_context.become and self._play_context.become_user not in ('root', remote_user):
             use_system_tmp = True
 
-        tmp_mode = None
-        if self._play_context.remote_user != 'root' or self._play_context.become and self._play_context.become_user != 'root':
-            tmp_mode = 0o755
+        tmp_mode = 0o700
 
         cmd = self._connection._shell.mkdtemp(basefile, use_system_tmp, tmp_mode)
-        display.debug("executing _low_level_execute_command to create the tmp path")
         result = self._low_level_execute_command(cmd, sudoable=False)
-        display.debug("done with creation of tmp path")
 
         # error handling on this seems a little aggressive?
         if result['rc'] != 0:
@@ -249,9 +251,11 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             cmd = self._connection._shell.remove(tmp_path, recurse=True)
             # If we have gotten here we have a working ssh configuration.
             # If ssh breaks we could leave tmp directories out on the remote system.
-            display.debug("calling _low_level_execute_command to remove the tmp path")
             self._low_level_execute_command(cmd, sudoable=False)
-            display.debug("done removing the tmp path")
+
+    def _transfer_file(self, local_path, remote_path):
+        self._connection.put_file(local_path, remote_path)
+        return remote_path
 
     def _transfer_data(self, remote_path, data):
         '''
@@ -267,54 +271,140 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             data = to_bytes(data, errors='strict')
             afo.write(data)
         except Exception as e:
-            #raise AnsibleError("failure encoding into utf-8: %s" % str(e))
             raise AnsibleError("failure writing module data to temporary file for transfer: %s" % str(e))
 
         afo.flush()
         afo.close()
 
         try:
-            self._connection.put_file(afile, remote_path)
+            self._transfer_file(afile, remote_path)
         finally:
             os.unlink(afile)
 
         return remote_path
 
-    def _remote_chmod(self, mode, path, sudoable=False):
+    def _fixup_perms(self, remote_path, remote_user, execute=False, recursive=True):
+        """
+        If the become_user is unprivileged and different from the
+        remote_user then we need to make the files we've uploaded readable by them.
+        """
+        if remote_path is None:
+            # Sometimes code calls us naively -- it has a var which could
+            # contain a path to a tmp dir but doesn't know if it needs to
+            # exist or not.  If there's no path, then there's no need for us
+            # to do work
+            self._display.debug('_fixup_perms called with remote_path==None.  Sure this is correct?')
+            return remote_path
+
+        if self._play_context.become and self._play_context.become_user not in ('root', remote_user):
+            # Unprivileged user that's different than the ssh user.  Let's get
+            # to work!
+            if remote_user == 'root':
+                # SSh'ing as root, therefore we can chown
+                self._remote_chown(remote_path, self._play_context.become_user, recursive=recursive)
+                if execute:
+                    # root can read things that don't have read bit but can't
+                    # execute them.
+                    self._remote_chmod('u+x', remote_path, recursive=recursive)
+            else:
+                if execute:
+                    mode = 'rx'
+                else:
+                    mode = 'rX'
+                # Try to use fs acls to solve this problem
+                res = self._remote_set_user_facl(remote_path, self._play_context.become_user, mode, recursive=recursive, sudoable=False)
+                if res['rc'] != 0:
+                    if C.ALLOW_WORLD_READABLE_TMPFILES:
+                        # fs acls failed -- do things this insecure way only
+                        # if the user opted in in the config file
+                        self._display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user')
+                        self._remote_chmod('a+%s' % mode, remote_path, recursive=recursive)
+                    else:
+                        raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user. For information on working around this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user')
+        elif execute:
+            # Can't depend on the file being transferred with execute
+            # permissions.  Only need user perms because no become was
+            # used here
+            self._remote_chmod('u+x', remote_path, recursive=recursive)
+
+        return remote_path
+
+    def _remote_chmod(self, mode, path, recursive=True, sudoable=False):
         '''
         Issue a remote chmod command
         '''
-
-        cmd = self._connection._shell.chmod(mode, path)
-        display.debug("calling _low_level_execute_command to chmod the remote path")
+        cmd = self._connection._shell.chmod(mode, path, recursive=recursive)
         res = self._low_level_execute_command(cmd, sudoable=sudoable)
-        display.debug("done with chmod call")
         return res
 
+    def _remote_chown(self, path, user, group=None, recursive=True, sudoable=False):
+        '''
+        Issue a remote chown command
+        '''
+        cmd = self._connection._shell.chown(path, user, group, recursive=recursive)
+        res = self._low_level_execute_command(cmd, sudoable=sudoable)
+        return res
+
+    def _remote_set_user_facl(self, path, user, mode, recursive=True, sudoable=False):
+        '''
+        Issue a remote call to setfacl
+        '''
+        cmd = self._connection._shell.set_user_facl(path, user, mode, recursive=recursive)
+        res = self._low_level_execute_command(cmd, sudoable=sudoable)
+        return res
+
+    def _execute_remote_stat(self, path, all_vars, follow, tmp=None):
+        '''
+        Get information from remote file.
+        '''
+        module_args=dict(
+           path=path,
+           follow=follow,
+           get_md5=False,
+           get_checksum=True,
+           checksum_algo='sha1',
+        )
+        mystat = self._execute_module(module_name='stat', module_args=module_args, task_vars=all_vars, tmp=tmp, delete_remote_tmp=(tmp is None))
+
+        if 'failed' in mystat and mystat['failed']:
+            raise AnsibleError('Failed to get information on remote file (%s): %s' % (path, mystat['msg']))
+
+        if not mystat['stat']['exists']:
+            # empty might be matched, 1 should never match, also backwards compatible
+            mystat['stat']['checksum'] = '1'
+
+        # happens sometimes when it is a dir and not on bsd
+        if not 'checksum' in mystat['stat']:
+            mystat['stat']['checksum'] = ''
+
+        return mystat['stat']
+
     def _remote_checksum(self, path, all_vars):
         '''
-        Takes a remote checksum and returns 1 if no file
+        Produces a remote checksum given a path,
+        Returns a number 0-4 for specific errors instead of checksum, also ensures it is different
+        0 = unknown error
+        1 = file does not exist, this might not be an error
+        2 = permissions issue
+        3 = its a directory, not a file
+        4 = stat module failed, likely due to not finding python
         '''
-
-        python_interp = all_vars.get('ansible_python_interpreter', 'python')
-
-        cmd = self._connection._shell.checksum(path, python_interp)
-        display.debug("calling _low_level_execute_command to get the remote checksum")
-        data = self._low_level_execute_command(cmd, sudoable=True)
-        display.debug("done getting the remote checksum")
+        x = "0" # unknown error has occured
         try:
-            data2 = data['stdout'].strip().splitlines()[-1]
-            if data2 == u'':
-                # this may happen if the connection to the remote server
-                # failed, so just return "INVALIDCHECKSUM" to avoid errors
-                return "INVALIDCHECKSUM"
+            remote_stat = self._execute_remote_stat(path, all_vars, follow=False)
+            if remote_stat['exists'] and remote_stat['isdir']:
+                x = "3" # its a directory not a file
             else:
-                return data2.split()[0]
-        except IndexError:
-            display.warning(u"Calculating checksum failed unusually, please report this to "
-                u"the list so it can be fixed\ncommand: %s\n----\noutput: %s\n----\n" % (to_unicode(cmd), data))
-            # this will signal that it changed and allow things to keep going
-            return "INVALIDCHECKSUM"
+                x = remote_stat['checksum'] # if 1, file is missing
+        except AnsibleError as e:
+            errormsg = to_unicode(e)
+            if errormsg.endswith('Permission denied'):
+                x = "2" # cannot read file
+            elif errormsg.endswith('MODULE FAILURE'):
+                x = "4" # python not found or module uncaught exception
+        finally:
+            return x
+
 
     def _remote_expand_user(self, path):
         ''' takes a remote path and performs tilde expansion on the remote host '''
@@ -329,9 +419,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
                 expand_path = '~%s' % self._play_context.become_user
 
         cmd = self._connection._shell.expand_user(expand_path)
-        display.debug("calling _low_level_execute_command to expand the remote user path")
         data = self._low_level_execute_command(cmd, sudoable=False)
-        display.debug("done expanding the remote user path")
         #initial_fragment = utils.last_non_blank_line(data['stdout'])
         initial_fragment = data['stdout'].strip().splitlines()[-1]
 
@@ -361,6 +449,14 @@ class ActionBase(with_metaclass(ABCMeta, object)):
 
         return data[idx:]
 
+    def _strip_success_message(self, data):
+        '''
+        Removes the BECOME-SUCCESS message from the data.
+        '''
+        if data.strip().startswith('BECOME-SUCCESS-'):
+            data = re.sub(r'^((\r)?\n)?BECOME-SUCCESS.*(\r)?\n', '', data)
+        return data
+
     def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, delete_remote_tmp=True):
         '''
         Transfer and run a module along with its arguments.
@@ -376,33 +472,42 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             module_args = self._task.args
 
         # set check mode in the module arguments, if required
-        if self._play_context.check_mode and not self._task.always_run:
+        if self._play_context.check_mode:
             if not self._supports_check_mode:
                 raise AnsibleError("check mode is not supported for this operation")
             module_args['_ansible_check_mode'] = True
+        else:
+            module_args['_ansible_check_mode'] = False
+
+        # Get the connection user for permission checks
+        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
 
         # set no log in the module arguments, if required
-        if self._play_context.no_log or not C.DEFAULT_NO_TARGET_SYSLOG:
-            module_args['_ansible_no_log'] = True
+        module_args['_ansible_no_log'] = self._play_context.no_log or C.DEFAULT_NO_TARGET_SYSLOG
 
         # set debug in the module arguments, if required
-        if C.DEFAULT_DEBUG:
-            module_args['_ansible_debug'] = True
+        module_args['_ansible_debug'] = C.DEFAULT_DEBUG
+
+        # let module know we are in diff mode
+        module_args['_ansible_diff'] = self._play_context.diff
+
+        # let module know our verbosity
+        module_args['_ansible_verbosity'] = self._display.verbosity
 
         (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
         if not shebang:
-            raise AnsibleError("module is missing interpreter line")
+            raise AnsibleError("module (%s) is missing interpreter line" % module_name)
 
         # a remote tmp path may be necessary and not already created
         remote_module_path = None
         args_file_path = None
         if not tmp and self._late_needs_tmp_path(tmp, module_style):
-            tmp = self._make_tmp_path()
+            tmp = self._make_tmp_path(remote_user)
 
         if tmp:
             remote_module_filename = self._connection._shell.get_remote_filename(module_name)
             remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename)
-            if module_style == 'old':
+            if module_style in ['old', 'non_native_want_json']:
                 # we'll also need a temp file to hold our module arguments
                 args_file_path = self._connection._shell.join_path(tmp, 'args')
 
@@ -416,18 +521,20 @@ class ActionBase(with_metaclass(ABCMeta, object)):
                 for k,v in iteritems(module_args):
                     args_data += '%s="%s" ' % (k, pipes.quote(text_type(v)))
                 self._transfer_data(args_file_path, args_data)
+            elif module_style == 'non_native_want_json':
+                self._transfer_data(args_file_path, json.dumps(module_args))
             display.debug("done transferring module to remote")
 
         environment_string = self._compute_environment_string()
 
-        if tmp and "tmp" in tmp and self._play_context.become and self._play_context.become_user != 'root':
-            # deal with possible umask issues once sudo'ed to other user
-            self._remote_chmod('a+r', remote_module_path)
+        # Fix permissions of the tmp path and tmp files.  This should be
+        # called after all files have been transferred.
+        self._fixup_perms(tmp, remote_user, recursive=True)
 
         cmd = ""
         in_data = None
 
-        if self._connection.has_pipelining and self._play_context.pipelining and not C.DEFAULT_KEEP_REMOTE_FILES:
+        if self._connection.has_pipelining and self._play_context.pipelining and not C.DEFAULT_KEEP_REMOTE_FILES and module_style == 'new':
             in_data = module_data
         else:
             if remote_module_path:
@@ -448,9 +555,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             # specified in the play, not the sudo_user
             sudoable = False
 
-        display.debug("calling _low_level_execute_command() for command %s" % cmd)
         res = self._low_level_execute_command(cmd, sudoable=sudoable, in_data=in_data)
-        display.debug("_low_level_execute_command returned ok")
 
         if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
             if self._play_context.become and self._play_context.become_user != 'root':
@@ -464,12 +569,12 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         except ValueError:
             # not valid json, lets try to capture error
             data = dict(failed=True, parsed=False)
-            if 'stderr' in res and res['stderr'].startswith(u'Traceback'):
-                data['exception'] = res['stderr']
-            else:
-                data['msg'] = res.get('stdout', u'')
-                if 'stderr' in res:
-                    data['msg'] += res['stderr']
+            data['msg'] = "MODULE FAILURE"
+            data['module_stdout'] = res.get('stdout', u'')
+            if 'stderr' in res:
+                data['module_stderr'] = res['stderr']
+                if res['stderr'].startswith(u'Traceback'):
+                    data['exception'] = res['stderr']
 
         # pre-split stdout into lines, if stdout is in the data and there
         # isn't already a stdout_lines value there
@@ -479,8 +584,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         display.debug("done with _execute_module (%s, %s)" % (module_name, module_args))
         return data
 
-    def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
-            executable=None, encoding_errors='replace'):
+    def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='replace'):
         '''
         This is the function which executes the low level shell command, which
         may be commands to create/remove directories for temporary files, or to
@@ -495,24 +599,25 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             replacement strategy (python3 could use surrogateescape)
         '''
 
-        if executable is not None:
-            cmd = executable + ' -c ' + cmd
-
-        display.debug("in _low_level_execute_command() (%s)" % (cmd,))
+        display.debug("_low_level_execute_command(): starting")
         if not cmd:
             # this can happen with powershell modules when there is no analog to a Windows command (like chmod)
-            display.debug("no command, exiting _low_level_execute_command()")
+            display.debug("_low_level_execute_command(): no command, exiting")
             return dict(stdout='', stderr='')
 
         allow_same_user = C.BECOME_ALLOW_SAME_USER
         same_user = self._play_context.become_user == self._play_context.remote_user
         if sudoable and self._play_context.become and (allow_same_user or not same_user):
-            display.debug("using become for this command")
+            display.debug("_low_level_execute_command(): using become for this command")
             cmd = self._play_context.make_become_cmd(cmd, executable=executable)
 
-        display.debug("executing the command %s through the connection" % cmd)
+        if self._connection.allow_executable:
+            if executable is None:
+                executable = self._play_context.executable
+            cmd = executable + ' -c ' + pipes.quote(cmd)
+
+        display.debug("_low_level_execute_command(): executing: %s" % (cmd,))
         rc, stdout, stderr = self._connection.exec_command(cmd, in_data=in_data, sudoable=sudoable)
-        display.debug("command execution done: rc=%s" % (rc))
 
         # stdout and stderr may be either a file-like or a bytes object.
         # Convert either one to a text type
@@ -530,11 +635,13 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         else:
             err = stderr
 
-        display.debug("stdout=%s, stderr=%s" % (stdout, stderr))
-        display.debug("done with _low_level_execute_command() (%s)" % (cmd,))
         if rc is None:
             rc = 0
 
+        # be sure to remove the BECOME-SUCCESS message now
+        out = self._strip_success_message(out)
+
+        display.debug("_low_level_execute_command() done: rc=%d, stdout=%s, stderr=%s" % (rc, stdout, stderr))
         return dict(rc=rc, stdout=out, stdout_lines=out.splitlines(), stderr=err)
 
     def _get_first_available_file(self, faf, of=None, searchdir='files'):
@@ -572,7 +679,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
                 diff['before'] = ''
             elif peek_result['appears_binary']:
                 diff['dst_binary'] = 1
-            elif peek_result['size'] > C.MAX_FILE_SIZE_FOR_DIFF:
+            elif C.MAX_FILE_SIZE_FOR_DIFF > 0 and peek_result['size'] > C.MAX_FILE_SIZE_FOR_DIFF:
                 diff['dst_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
             else:
                 display.debug("Slurping the file %s" % source)
@@ -587,23 +694,31 @@ class ActionBase(with_metaclass(ABCMeta, object)):
                     diff['before'] = dest_contents
 
             if source_file:
-                display.debug("Reading local copy of the file %s" % source)
-                try:
-                    src = open(source)
-                    src_contents = src.read(8192)
-                    st = os.stat(source)
-                except Exception as e:
-                    raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, str(e)))
-                if "\x00" in src_contents:
-                    diff['src_binary'] = 1
-                elif st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
+                st = os.stat(source)
+                if C.MAX_FILE_SIZE_FOR_DIFF > 0 and st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
                     diff['src_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
                 else:
-                    diff['after_header'] = source
-                    diff['after'] = src_contents
+                    display.debug("Reading local copy of the file %s" % source)
+                    try:
+                        src = open(source)
+                        src_contents = src.read()
+                    except Exception as e:
+                        raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, str(e)))
+
+                    if "\x00" in src_contents:
+                        diff['src_binary'] = 1
+                    else:
+                        diff['after_header'] = source
+                        diff['after'] = src_contents
             else:
                 display.debug("source of file passed in")
                 diff['after_header'] = 'dynamically generated'
                 diff['after'] = source
 
+        if self._play_context.no_log:
+            if 'before' in diff:
+                diff["before"] = ""
+            if 'after' in diff:
+                diff["after"] = " [[ Diff output has been hidden because 'no_log: true' was specified for this result ]]"
+
         return diff

lib/ansible/plugins/action/add_host.py

@@ -53,9 +53,13 @@ class ActionModule(ActionBase):
         new_name = self._task.args.get('name', self._task.args.get('hostname', None))
         display.vv("creating host via 'add_host': hostname=%s" % new_name)
 
-        name, port = parse_address(new_name, allow_ranges=False)
-        if not name:
-            raise AnsibleError("Invalid inventory hostname: %s" % new_name)
+        try:
+            name, port = parse_address(new_name, allow_ranges=False)
+        except:
+            # not a parsable hostname, but might still be usable
+            name = new_name
+            port = None
+
         if port:
             self._task.args['ansible_ssh_port'] = port
 

lib/ansible/plugins/action/assemble.py

@@ -89,6 +89,7 @@ class ActionModule(ActionBase):
         delimiter  = self._task.args.get('delimiter', None)
         remote_src = self._task.args.get('remote_src', 'yes')
         regexp     = self._task.args.get('regexp', None)
+        follow     = self._task.args.get('follow', False)
         ignore_hidden = self._task.args.get('ignore_hidden', False)
 
         if src is None or dest is None:
@@ -96,10 +97,17 @@ class ActionModule(ActionBase):
             result['msg'] = "src and dest are required"
             return result
 
-        if boolean(remote_src):
-            result.update(self._execute_module(tmp=tmp, task_vars=task_vars))
-            return result
+        cleanup_remote_tmp = False
+        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
+        if not tmp:
+            tmp = self._make_tmp_path(remote_user)
+            cleanup_remote_tmp = True
 
+        if boolean(remote_src):
+            result.update(self._execute_module(tmp=tmp, task_vars=task_vars, delete_remote_tmp=False))
+            if cleanup_remote_tmp:
+                self._remove_tmp_path(tmp)
+            return result
         elif self._task._role is not None:
             src = self._loader.path_dwim_relative(self._task._role._role_path, 'files', src)
         else:
@@ -109,51 +117,56 @@ class ActionModule(ActionBase):
         if regexp is not None:
             _re = re.compile(regexp)
 
+        if not os.path.isdir(src):
+            result['failed'] = True
+            result['msg'] = "Source (%s) is not a directory" % src
+            return result
+
         # Does all work assembling the file
         path = self._assemble_from_fragments(src, delimiter, _re, ignore_hidden)
 
         path_checksum = checksum_s(path)
         dest = self._remote_expand_user(dest)
-        remote_checksum = self._remote_checksum(dest, all_vars=task_vars)
+        dest_stat = self._execute_remote_stat(dest, all_vars=task_vars, follow=follow, tmp=tmp)
 
         diff = {}
-        if path_checksum != remote_checksum:
-            resultant = file(path).read()
+
+        # setup args for running modules
+        new_module_args = self._task.args.copy()
+
+        # clean assemble specific options
+        for opt in ['remote_src', 'regexp', 'delimiter', 'ignore_hidden']:
+            if opt in new_module_args:
+                del new_module_args[opt]
+
+        new_module_args.update(
+            dict(
+                dest=dest,
+                original_basename=os.path.basename(src),
+            )
+        )
+
+        if path_checksum != dest_stat['checksum']:
 
             if self._play_context.diff:
                 diff = self._get_diff_data(dest, path, task_vars)
 
-            xfered = self._transfer_data('src', resultant)
+            remote_path = self._connection._shell.join_path(tmp, 'src')
+            xfered = self._transfer_file(path, remote_path)
 
             # fix file permissions when the copy is done as a different user
-            if self._play_context.become and self._play_context.become_user != 'root':
-                self._remote_chmod('a+r', xfered)
+            self._fixup_perms(tmp, remote_user, recursive=True)
 
-            # run the copy module
+            new_module_args.update( dict( src=xfered,))
 
-            new_module_args = self._task.args.copy()
-            new_module_args.update(
-                dict(
-                    src=xfered,
-                    dest=dest,
-                    original_basename=os.path.basename(src),
-                )
-            )
-
-            res = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp)
+            res = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False)
             if diff:
                 res['diff'] = diff
             result.update(res)
-            return result
         else:
-            new_module_args = self._task.args.copy()
-            new_module_args.update(
-                dict(
-                    src=xfered,
-                    dest=dest,
-                    original_basename=os.path.basename(src),
-                )
-            )
+            result.update(self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False))
 
-            result.update(self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp))
-            return result
+        if tmp and cleanup_remote_tmp:
+            self._remove_tmp_path(tmp)
+
+        return result

lib/ansible/plugins/action/async.py

@@ -38,8 +38,9 @@ class ActionModule(ActionBase):
             result['msg'] = 'check mode not supported for this module'
             return result
 
+        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
         if not tmp:
-            tmp = self._make_tmp_path()
+            tmp = self._make_tmp_path(remote_user)
 
         module_name = self._task.action
         async_module_path  = self._connection._shell.join_path(tmp, 'async_wrapper')
@@ -48,21 +49,31 @@ class ActionModule(ActionBase):
         env_string = self._compute_environment_string()
 
         module_args = self._task.args.copy()
-        if self._play_context.no_log or not C.DEFAULT_NO_TARGET_SYSLOG:
+        if self._play_context.no_log or C.DEFAULT_NO_TARGET_SYSLOG:
             module_args['_ansible_no_log'] = True
 
         # configure, upload, and chmod the target module
         (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
         self._transfer_data(remote_module_path, module_data)
-        self._remote_chmod('a+rx', remote_module_path)
 
         # configure, upload, and chmod the async_wrapper module
         (async_module_style, shebang, async_module_data) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars)
         self._transfer_data(async_module_path, async_module_data)
-        self._remote_chmod('a+rx', async_module_path)
 
         argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), json.dumps(module_args))
 
+        self._fixup_perms(tmp, remote_user, execute=True, recursive=True)
+        # Only the following two files need to be executable but we'd have to
+        # make three remote calls if we wanted to just set them executable.
+        # There's not really a problem with marking too many of the temp files
+        # executable so we go ahead and mark them all as executable in the
+        # line above (the line above is needed in any case [although
+        # execute=False is okay if we uncomment the lines below] so that all
+        # the files are readable in case the remote_user and become_user are
+        # different and both unprivileged)
+        #self._fixup_perms(remote_module_path, remote_user, execute=True, recursive=False)
+        #self._fixup_perms(async_module_path, remote_user, execute=True, recursive=False)
+
         async_limit = self._task.async
         async_jid   = str(random.randint(0, 999999999999))
 

lib/ansible/plugins/action/copy.py

@@ -46,6 +46,7 @@ class ActionModule(ActionBase):
         force   = boolean(self._task.args.get('force', 'yes'))
         faf     = self._task.first_available_file
         remote_src = boolean(self._task.args.get('remote_src', False))
+        follow  = boolean(self._task.args.get('follow', False))
 
         if (source is None and content is None and faf is None) or dest is None:
             result['failed'] = True
@@ -106,7 +107,7 @@ class ActionModule(ActionBase):
         source_files = []
 
         # If source is a directory populate our list else source is a file and translate it to a tuple.
-        if os.path.isdir(source):
+        if os.path.isdir(to_bytes(source, errors='strict')):
             # Get the amount of spaces to remove to get the relative path.
             if source_trailing_slash:
                 sz = len(source)
@@ -140,9 +141,10 @@ class ActionModule(ActionBase):
         delete_remote_tmp = (len(source_files) == 1)
 
         # If this is a recursive action create a tmp path that we can share as the _exec_module create is too late.
+        remote_user = task_vars.get('ansible_ssh_user') or self._play_context.remote_user
         if not delete_remote_tmp:
             if tmp is None or "-tmp-" not in tmp:
-                tmp = self._make_tmp_path()
+                tmp = self._make_tmp_path(remote_user)
 
         # expand any user home dir specifier
         dest = self._remote_expand_user(dest)
@@ -167,11 +169,11 @@ class ActionModule(ActionBase):
             else:
                 dest_file = self._connection._shell.join_path(dest)
 
-            # Attempt to get the remote checksum
-            remote_checksum = self._remote_checksum(dest_file, all_vars=task_vars)
+            # Attempt to get remote file info
+            dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, tmp=tmp)
 
-            if remote_checksum == '3':
-                # The remote_checksum was executed on a directory.
+            if dest_status['exists'] and dest_status['isdir']:
+                # The dest is a directory.
                 if content is not None:
                     # If source was defined as content remove the temporary file and fail out.
                     self._remove_tempfile_if_content_defined(content, content_tempfile)
@@ -179,15 +181,15 @@ class ActionModule(ActionBase):
                     result['msg'] = "can not use content with a dir as dest"
                     return result
                 else:
-                    # Append the relative source location to the destination and retry remote_checksum
+                    # Append the relative source location to the destination and get remote stats again
                     dest_file = self._connection._shell.join_path(dest, source_rel)
-                    remote_checksum = self._remote_checksum(dest_file, all_vars=task_vars)
+                    dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, tmp=tmp)
 
-            if remote_checksum != '1' and not force:
+            if dest_status['exists'] and not force:
                 # remote_file does not exist so continue to next iteration.
                 continue
 
-            if local_checksum != remote_checksum:
+            if local_checksum != dest_status['checksum']:
                 # The checksums don't match and we will change or error out.
                 changed = True
 
@@ -195,7 +197,7 @@ class ActionModule(ActionBase):
                 # If this is recursive we already have a tmp path.
                 if delete_remote_tmp:
                     if tmp is None or "-tmp-" not in tmp:
-                        tmp = self._make_tmp_path()
+                        tmp = self._make_tmp_path(remote_user)
 
                 if self._play_context.diff and not raw:
                     diffs.append(self._get_diff_data(dest_file, source_full, task_vars))
@@ -210,16 +212,15 @@ class ActionModule(ActionBase):
                 tmp_src = self._connection._shell.join_path(tmp, 'source')
 
                 if not raw:
-                    self._connection.put_file(source_full, tmp_src)
+                    self._transfer_file(source_full, tmp_src)
                 else:
-                    self._connection.put_file(source_full, dest_file)
+                    self._transfer_file(source_full, dest_file)
 
                 # We have copied the file remotely and no longer require our content_tempfile
                 self._remove_tempfile_if_content_defined(content, content_tempfile)
 
                 # fix file permissions when the copy is done as a different user
-                if self._play_context.become and self._play_context.become_user != 'root':
-                    self._remote_chmod('a+r', tmp_src)
+                self._fixup_perms(tmp, remote_user, recursive=True)
 
                 if raw:
                     # Continue to next iteration if raw is defined.

lib/ansible/plugins/action/debug.py

@@ -20,38 +20,55 @@ __metaclass__ = type
 from ansible.plugins.action import ActionBase
 from ansible.utils.boolean import boolean
 from ansible.utils.unicode import to_unicode
+from ansible.errors import AnsibleUndefinedVariable
 
 class ActionModule(ActionBase):
     ''' Print statements during execution '''
 
     TRANSFERS_FILES = False
+    VALID_ARGS = set(['msg', 'var', 'verbosity'])
 
     def run(self, tmp=None, task_vars=None):
         if task_vars is None:
             task_vars = dict()
 
+        for arg in self._task.args:
+            if arg not in self.VALID_ARGS:
+                return {"failed": True, "msg": "'%s' is not a valid option in debug" % arg}
+
+        if 'msg' in self._task.args and 'var' in self._task.args:
+            return {"failed": True, "msg": "'msg' and 'var' are incompatible options"}
+
         result = super(ActionModule, self).run(tmp, task_vars)
 
-        if 'msg' in self._task.args:
-            if 'fail' in self._task.args and boolean(self._task.args['fail']):
-                result['failed'] = True
-                result['msg'] = self._task.args['msg']
-            else:
-                result['msg'] = self._task.args['msg']
-        # FIXME: move the LOOKUP_REGEX somewhere else
-        elif 'var' in self._task.args: # and not utils.LOOKUP_REGEX.search(self._task.args['var']):
-            results = self._templar.template(self._task.args['var'], convert_bare=True)
-            if type(self._task.args['var']) in (list, dict):
-                # If var is a list or dict, use the type as key to display
-                result[to_unicode(type(self._task.args['var']))] = results
-            else:
-                if results == self._task.args['var']:
-                    results = "VARIABLE IS NOT DEFINED!"
-                result[self._task.args['var']] = results
-        else:
-            result['msg'] = 'here we are'
+        verbosity = 0
+        # get task verbosity
+        if 'verbosity' in self._task.args:
+            verbosity = int(self._task.args['verbosity'])
 
-        # force flag to make debug output module always verbose
-        result['_ansible_verbose_always'] = True
+        if verbosity <= self._display.verbosity:
+            if 'msg' in self._task.args:
+                result['msg'] = self._task.args['msg']
+
+            elif 'var' in self._task.args:
+                try:
+                    results = self._templar.template(self._task.args['var'], convert_bare=True, fail_on_undefined=True, bare_deprecated=False)
+                    if results == self._task.args['var']:
+                        raise AnsibleUndefinedVariable
+                except AnsibleUndefinedVariable:
+                    results = "VARIABLE IS NOT DEFINED!"
+
+                if type(self._task.args['var']) in (list, dict):
+                    # If var is a list or dict, use the type as key to display
+                    result[to_unicode(type(self._task.args['var']))] = results
+                else:
+                    result[self._task.args['var']] = results
+            else:
+                result['msg'] = 'Hello world!'
+
+            # force flag to make debug output module always verbose
+            result['_ansible_verbose_always'] = True
+        else:
+            result['skipped'] = True
 
         return result

lib/ansible/plugins/action/eos_template.py

@@ -0,0 +1,26 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.net_template import ActionModule as NetActionModule
+
+class ActionModule(NetActionModule, ActionBase):
+    pass

lib/ansible/plugins/action/fetch.py

@@ -25,6 +25,7 @@ from ansible.plugins.action import ActionBase
 from ansible.utils.boolean import boolean
 from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
 from ansible.utils.path import makedirs_safe
+from ansible.utils.unicode import to_bytes
 
 
 class ActionModule(ActionBase):
@@ -70,7 +71,7 @@ class ActionModule(ActionBase):
         if remote_checksum in ('1', '2', None):
             slurpres = self._execute_module(module_name='slurp', module_args=dict(src=source), task_vars=task_vars, tmp=tmp)
             if slurpres.get('failed'):
-                if remote_checksum == '1' and not fail_on_missing:
+                if not fail_on_missing and (slurpres.get('msg').startswith('file not found') or remote_checksum == '1'):
                     result['msg'] = "the remote file does not exist, not transferring, ignored"
                     result['file'] = source
                     result['changed'] = False
@@ -158,7 +159,7 @@ class ActionModule(ActionBase):
                 self._connection.fetch_file(source, dest)
             else:
                 try:
-                    f = open(dest, 'w')
+                    f = open(to_bytes(dest, errors='strict'), 'w')
                     f.write(remote_data)
                     f.close()
                 except (IOError, OSError) as e:
@@ -171,7 +172,9 @@ class ActionModule(ActionBase):
                 new_md5 = None
 
             if validate_checksum and new_checksum != remote_checksum:
-                result.update(dict(failed=True, md5sum=new_md5, msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum))
+                result.update(dict(failed=True, md5sum=new_md5,
+                    msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None,
+                    checksum=new_checksum, remote_checksum=remote_checksum))
             else:
                 result.update(dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum))
         else:

lib/ansible/plugins/action/group_by.py

@@ -40,6 +40,6 @@ class ActionModule(ActionBase):
         group_name = self._task.args.get('key')
         group_name = group_name.replace(' ','-')
 
-        result['changed'] = True
+        result['changed'] = False
         result['add_group'] = group_name
         return result

lib/ansible/plugins/action/ios_template.py

@@ -0,0 +1,28 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.net_template import ActionModule as NetActionModule
+
+class ActionModule(NetActionModule, ActionBase):
+    pass
+
+

lib/ansible/plugins/action/iosxr_template.py

@@ -0,0 +1,28 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.net_template import ActionModule as NetActionModule
+
+class ActionModule(NetActionModule, ActionBase):
+    pass
+
+

lib/ansible/plugins/action/junos_template.py

@@ -0,0 +1,28 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.net_template import ActionModule as NetActionModule
+
+class ActionModule(NetActionModule, ActionBase):
+    pass
+
+

lib/ansible/plugins/action/net_template.py

@@ -0,0 +1,98 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import sys
+import os
+import time
+import glob
+import urlparse
+
+from ansible.plugins.action import ActionBase
+from ansible.utils.boolean import boolean
+from ansible.utils.unicode import to_unicode
+
+BOOLEANS = ('true', 'false', 'yes', 'no')
+
+class ActionModule(ActionBase):
+
+    TRANSFERS_FILES = False
+
+    def run(self, tmp=None, task_vars=None):
+        result = super(ActionModule, self).run(tmp, task_vars)
+        result['changed'] = False
+
+        try:
+            self._handle_template()
+        except ValueError as exc:
+            return dict(failed=True, msg=exc.message)
+
+        result.update(self._execute_module(module_name=self._task.action,
+            module_args=self._task.args, task_vars=task_vars))
+
+        if self._task.args.get('backup') and result.get('_backup'):
+            # User requested backup and no error occurred in module.
+            # NOTE: If there is a parameter error, _backup key may not be in results.
+            self._write_backup(task_vars['inventory_hostname'], result['_backup'])
+
+        if '_backup' in result:
+            del result['_backup']
+
+        return result
+
+    def _get_working_path(self):
+        cwd = self._loader.get_basedir()
+        if self._task._role is not None:
+            cwd = self._task._role._role_path
+        return cwd
+
+    def _write_backup(self, host, contents):
+        backup_path = self._get_working_path() + '/backup'
+        if not os.path.exists(backup_path):
+            os.mkdir(backup_path)
+        for fn in glob.glob('%s/%s*' % (backup_path, host)):
+            os.remove(fn)
+        tstamp = time.strftime("%Y-%m-%d@%H:%M:%S", time.localtime(time.time()))
+        filename = '%s/%s_config.%s' % (backup_path, host, tstamp)
+        open(filename, 'w').write(contents)
+
+    def _handle_template(self):
+        src = self._task.args.get('src')
+        working_path = self._get_working_path()
+
+        if os.path.isabs(src) or urlparse.urlsplit('src').scheme:
+            source = src
+        else:
+            source = self._loader.path_dwim_relative(working_path, 'templates', src)
+            if not source:
+                source = self._loader.path_dwim_relative(working_path, src)
+
+        if not os.path.exists(source):
+            return
+
+        try:
+            with open(source, 'r') as f:
+                template_data = to_unicode(f.read())
+        except IOError:
+            return dict(failed=True, msg='unable to load src file')
+
+        self._task.args['src'] = self._templar.template(template_data)
+
+

lib/ansible/plugins/action/normal.py

@@ -18,6 +18,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from ansible.plugins.action import ActionBase
+from ansible.utils.vars import merge_hash
 
 
 class ActionModule(ActionBase):
@@ -27,12 +28,16 @@ class ActionModule(ActionBase):
             task_vars = dict()
 
         results = super(ActionModule, self).run(tmp, task_vars)
-        results.update(self._execute_module(tmp=tmp, task_vars=task_vars))
-
+        # remove as modules might hide due to nolog
+        del results['invocation']['module_args']
+        results = merge_hash(results, self._execute_module(tmp=tmp, task_vars=task_vars))
         # Remove special fields from the result, which can only be set
         # internally by the executor engine. We do this only here in
         # the 'normal' action, as other action plugins may set this.
-        for field in ('ansible_notify',):
+        #
+        # We don't want modules to determine that running the module fires
+        # notify handlers.  That's for the playbook to decide.
+        for field in ('_ansible_notify',):
             if field in results:
                 results.pop(field)
 

lib/ansible/plugins/action/nxos_template.py

@@ -0,0 +1,27 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.net_template import ActionModule as NetActionModule
+
+class ActionModule(NetActionModule, ActionBase):
+    pass
+

lib/ansible/plugins/action/ops_template.py

@@ -0,0 +1,50 @@
+#
+# Copyright 2015 Peter Sprygada <psprygada@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from ansible.plugins.action import ActionBase
+from ansible.plugins.action.net_template import ActionModule as NetActionModule
+
+class ActionModule(NetActionModule, ActionBase):
+
+    def run(self, tmp=None, task_vars=None):
+        if self._connection.transport == 'local':
+            return super(ActionModule, self).run(tmp, task_vars)
+
+        result = dict(changed=False)
+
+        if isinstance(self._task.args['src'], basestring):
+            self._handle_template()
+
+        result.update(self._execute_module(module_name=self._task.action,
+            module_args=self._task.args, task_vars=task_vars))
+
+        if self._task.args.get('backup') and result.get('_backup'):
+            contents = json.dumps(result['_backup'], indent=4)
+            self._write_backup(task_vars['inventory_hostname'], contents)
+
+        if '_backup' in result:
+            del result['_backup']
+
+        return result
+
+