Merge branch 'master' into localconnection

Merge the SortedOptParser bits and debug attribute commits into
localconnection.

Conflicts:
	bin/ansible
	lib/ansible/playbook.py
	lib/ansible/runner.py
	lib/ansible/utils.py

lib/ansible/callbacks.py

@@ -95,6 +95,9 @@ class DefaultRunnerCallbacks(object):
     def on_unreachable(self, host, res):
         pass
 
+    def on_no_hosts(self):
+        pass
+
 ########################################################################
 
 class CliRunnerCallbacks(DefaultRunnerCallbacks):
@@ -120,6 +123,9 @@ class CliRunnerCallbacks(DefaultRunnerCallbacks):
 
     def on_error(self, host, err):
         print >>sys.stderr, "stderr: [%s] => %s\n" % (host, err)
+    
+    def on_no_hosts(self):
+        print >>sys.stderr, "no hosts matched\n"
 
     def _on_any(self, host, result):
         print utils.host_report_msg(host, self.options.module_name, result, self.options.one_line)
@@ -159,6 +165,9 @@ class PlaybookRunnerCallbacks(DefaultRunnerCallbacks):
     def on_skipped(self, host):
         print "skipping: [%s]\n" % host
 
+    def on_no_hosts(self):
+        print "no hosts matched or remaining\n"
+
 ########################################################################
 
 class PlaybookCallbacks(object):

lib/ansible/connection.py

@@ -18,7 +18,13 @@
 
 ################################################
 
-import paramiko
+import warnings
+# prevent paramiko warning noise
+# see http://stackoverflow.com/questions/3920502/
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    import paramiko
+
 import traceback
 import os
 import time
@@ -142,6 +148,15 @@ class ParamikoConnection(object):
             raise errors.AnsibleError("failed to transfer file to %s" % out_path)
         sftp.close()
 
+    def fetch_file(self, in_path, out_path):
+        sftp = self.ssh.open_sftp()
+        try:
+            sftp.get(in_path, out_path)
+        except IOError:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to transfer file from %s" % in_path)
+        sftp.close()
+
     def close(self):
         ''' terminate the connection '''
 
@@ -184,6 +199,10 @@ class LocalConnection(object):
             traceback.print_exc()
             raise errors.AnsibleError("failed to transfer file to %s" % out_path)
 
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from local to local -- for copatibility '''
+        self.put_file(in_path, out_path)
+
     def close(self):
         ''' terminate the connection; nothing to do here '''
 

lib/ansible/playbook.py

@@ -439,24 +439,17 @@ class PlayBook(object):
         else:
             self.callbacks.on_setup_primary()
 
-        # first run the setup task on every node, which gets the variables
-        # written to the JSON file and will also bubble facts back up via
-        # magic in Runner()
-        push_var_str=''
-        for (k,v) in vars.iteritems():
-            push_var_str += "%s=\"%s\" " % (k,v)
-
         host_list = [ h for h in self.host_list if not (h in self.stats.failures or h in self.stats.dark) ]
 
         # push any variables down to the system
         setup_results = ansible.runner.Runner(
             pattern=pattern, groups=self.groups, module_name='setup',
-            module_args=push_var_str, host_list=host_list,
+            module_args=vars, host_list=host_list,
             forks=self.forks, module_path=self.module_path,
             timeout=self.timeout, remote_user=user,
             remote_pass=self.remote_pass, remote_port=self.remote_port,
             setup_cache=SETUP_CACHE,
-            callbacks=self.runner_callbacks, sudo=sudo,
+            callbacks=self.runner_callbacks, sudo=sudo, debug=self.debug,
             transport=transport,
         ).run()
         self.stats.compute(setup_results, setup=True)

lib/ansible/runner.py

@@ -119,8 +119,8 @@ class Runner(object):
         euid = pwd.getpwuid(os.geteuid())[0]
         if self.transport == 'local' and self.remote_user != euid:
             raise Exception("User mismatch: expected %s, but is %s" % (self.remote_user, euid))
-        if type(self.module_args) != str:
-            raise Exception("module_args must be a string: %s" % self.module_args)
+        if type(self.module_args) != str and type(self.module_args) != dict:
+            raise Exception("module_args must be a string or dict: %s" % self.module_args)
 
         self._tmp_paths  = {}
         random.seed()
@@ -277,6 +277,9 @@ class Runner(object):
     def _transfer_str(self, conn, tmp, name, args_str):
         ''' transfer arguments as a single file to be fed to the module. '''
 
+        if type(args_str) == dict:
+            args_str = utils.smjson(args_str)
+
         args_fd, args_file = tempfile.mkstemp()
         args_fo = os.fdopen(args_fd, 'w')
         args_fo.write(args_str)
@@ -322,23 +325,43 @@ class Runner(object):
     def _add_setup_vars(self, inject, args):
         ''' setup module variables need special handling '''
 
+        is_dict = False
+        if type(args) == dict:
+            is_dict = True
+
+        # TODO: keep this as a dict through the whole path to simplify this code
         for (k,v) in inject.iteritems():
             if not k.startswith('facter_') and not k.startswith('ohai_'):
-                if str(v).find(" ") != -1:
-                    v = "\"%s\"" % v
-                args += " %s=%s" % (k, str(v).replace(" ","~~~"))
+                if not is_dict:
+                    if str(v).find(" ") != -1:
+                        v = "\"%s\"" % v
+                    args += " %s=%s" % (k, str(v).replace(" ","~~~"))
+                else:
+                    args[k]=v
         return args   
  
     # *****************************************************
 
     def _add_setup_metadata(self, args):
         ''' automatically determine where to store variables for the setup module '''
+        
+        is_dict = False
+        if type(args) == dict:
+            is_dict = True
 
-        if args.find("metadata=") == -1:
-            if self.remote_user == 'root':
-                args = "%s metadata=/etc/ansible/setup" % args
-            else:
-                args = "%s metadata=/home/%s/.ansible/setup" % (args, self.remote_user)
+        # TODO: keep this as a dict through the whole path to simplify this code
+        if not is_dict:
+            if args.find("metadata=") == -1:
+                if self.remote_user == 'root':
+                    args = "%s metadata=/etc/ansible/setup" % args
+                else:
+                    args = "%s metadata=/home/%s/.ansible/setup" % (args, self.remote_user)
+        else:
+            if not 'metadata' in args:
+                if self.remote_user == 'root':
+                    args['metadata'] = '/etc/ansible/setup'
+                else:
+                    args['metadata'] = "/home/%s/.ansible/setup" % (self.remote_user)
         return args   
  
     # *****************************************************
@@ -358,9 +381,11 @@ class Runner(object):
             args = self._add_setup_vars(inject, args)
             args = self._add_setup_metadata(args)
 
+        if type(args) == dict:
+           args = utils.bigjson(args)
         args = utils.template(args, inject)
+
         module_name_tail = remote_module_path.split("/")[-1]
-        client_executed_str = "%s %s" % (module_name_tail, args.strip())
 
         argsfile = self._transfer_str(conn, tmp, 'arguments', args)
         if async_jid is None:
@@ -368,12 +393,8 @@ class Runner(object):
         else:
             cmd = " ".join([str(x) for x in [remote_module_path, async_jid, async_limit, async_module, argsfile]])
 
-        # log command as the full command not as the path to args file - helps with debugging
-        msg = '%s: "%s"' % (self.module_name, args)
-        conn.exec_command('/usr/bin/logger -t ansible -p auth.info "%s"' % msg, None)
-
-                    
         res, err = self._exec_command(conn, cmd, tmp, sudoable=True)
+        client_executed_str = "%s %s" % (module_name_tail, args.strip())
         return ( res, err, client_executed_str )
 
     # *****************************************************
@@ -443,8 +464,10 @@ class Runner(object):
 
         # load up options
         options = utils.parse_kv(self.module_args)
-        source = options['src']
-        dest   = options['dest']
+        source = options.get('src', None)
+        dest   = options.get('dest', None)
+        if source is None or dest is None:
+            return (host, True, dict(failed=True, msg="src and dest are required"), '')
         
         # transfer the file to a remote tmp location
         tmp_src = tmp + source.split('/')[-1]
@@ -466,6 +489,42 @@ class Runner(object):
 
     # *****************************************************
 
+    def _execute_fetch(self, conn, host, tmp):
+        ''' handler for fetch operations '''
+
+        # load up options
+        options = utils.parse_kv(self.module_args)
+        source = options.get('src', None)
+        dest = options.get('dest', None)
+        if source is None or dest is None:
+            return (host, True, dict(failed=True, msg="src and dest are required"), '')
+
+        # files are saved in dest dir, with a subdir for each host, then the filename
+        filename = os.path.basename(source)
+        dest   = "%s/%s/%s" % (utils.path_dwim(self.basedir, dest), host, filename)
+
+        # compare old and new md5 for support of change hooks
+        local_md5 = None
+        if os.path.exists(dest):
+            local_md5 = os.popen("md5sum %s" % dest).read().split()[0]
+        remote_md5 = self._exec_command(conn, "md5sum %s" % source, tmp, True)[0].split()[0]
+
+        if remote_md5 != local_md5:
+            # create the containing directories, if needed
+            os.makedirs(os.path.dirname(dest))
+            # fetch the file and check for changes
+            conn.fetch_file(source, dest)
+            new_md5 = os.popen("md5sum %s" % dest).read().split()[0]
+            changed = (new_md5 != local_md5)
+            if new_md5 != remote_md5:
+                return (host, True, dict(failed=True, msg="md5 mismatch", md5sum=new_md5), '')
+            return (host, True, dict(changed=True, md5sum=new_md5), '')
+        else:
+            return (host, True, dict(changed=False, md5sum=local_md5), '')
+        
+        
+    # *****************************************************
+
     def _chain_file_module(self, conn, tmp, data, err, options, executed):
         ''' handles changing file attribs after copy/template operations '''
 
@@ -488,9 +547,11 @@ class Runner(object):
 
         # load up options
         options  = utils.parse_kv(self.module_args)
-        source   = options['src']
-        dest     = options['dest']
+        source   = options.get('src', None)
+        dest     = options.get('dest', None)
         metadata = options.get('metadata', None)
+        if source is None or dest is None:
+            return (host, True, dict(failed=True, msg="src and dest are required"), '')
 
         if metadata is None:
             if self.remote_user == 'root':
@@ -555,6 +616,8 @@ class Runner(object):
 
         if self.module_name == 'copy':
             result = self._execute_copy(conn, host, tmp)
+        elif self.module_name == 'fetch':
+            result = self._execute_fetch(conn, host, tmp)
         elif self.module_name == 'template':
             result = self._execute_template(conn, host, tmp)
         else:
@@ -587,10 +650,6 @@ class Runner(object):
     def _exec_command(self, conn, cmd, tmp, sudoable=False):
         ''' execute a command string over SSH, return the output '''
 
-        msg = '%s: %s' % (self.module_name, cmd)
-        # log remote command execution
-        conn.exec_command('/usr/bin/logger -t ansible -p auth.info "%s"' % msg, None)
-        # now run actual command
         stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudoable=sudoable)
 
         if type(stderr) != str:
@@ -697,6 +756,7 @@ class Runner(object):
         # find hosts that match the pattern
         hosts = self._match_hosts(self.pattern)
         if len(hosts) == 0:
+            self.callbacks.on_no_hosts()
             return dict(contacted={}, dark={})
  
         hosts = [ (self,x) for x in hosts ]

lib/ansible/utils.py

@@ -24,7 +24,7 @@ import re
 import jinja2
 import yaml
 import optparse
-
+from operator import methodcaller
 try:
     import json
 except ImportError:
@@ -273,79 +273,55 @@ def parse_kv(args):
             options[k]=v
     return options
 
-def make_parser(add_options, constants=C, usage="", output_opts=False, runas_opts=False, async_opts=False, connect_opts=False):
-    ''' create an options parser w/ common options for any ansible program '''
+class SortedOptParser(optparse.OptionParser):
+    '''Optparser which sorts the options by opt before outputting --help'''
+    def format_help(self, formatter=None):
+        self.option_list.sort(key=methodcaller('get_opt_string'))
+        return optparse.OptionParser.format_help(self, formatter=None)
 
-    options = base_parser_options(
-        constants=constants, 
-        output_opts=output_opts, 
-        runas_opts=runas_opts,
-        async_opts=async_opts,
-        connect_opts=connect_opts
-    )
-    options.update(add_options)
+def base_parser(constants=C, usage="", output_opts=False, runas_opts=False, async_opts=False, connect_opts=False):
+    ''' create an options parser for any ansible script '''
 
-    parser = optparse.OptionParser()
-    names = sorted(options.keys())
-    for n in names:
-        data  = options[n].copy()
-        long  = data['long']
-        del data['long']
-        parser.add_option(n, long, **data)
-    return parser
-
-def base_parser_options(constants=C, output_opts=False, runas_opts=False, async_opts=False, connect_opts=False):
-    ''' creates common options for ansible programs '''
-
-    options = {
-        '-D': dict(long='--debug', default=False, action="store_true",
-                   help='show debug/verbose module output'),
-        '-f': dict(long='--forks', dest='forks', default=constants.DEFAULT_FORKS, type='int',
-                   help='number of parallel processes to use'),
-        '-i': dict(long='--inventory-file', dest='inventory',
-                   help='path to inventory host file', default=constants.DEFAULT_HOST_LIST),
-        '-k': dict(long='--ask-pass', default=False, action='store_true',
-                   help='ask for SSH password'),
-        '-M': dict(long='--module-path', dest='module_path',
-                   help="path to module library directory", default=constants.DEFAULT_MODULE_PATH),
-        '-T': dict(long='--timeout', default=constants.DEFAULT_TIMEOUT, type='int',
-                   dest='timeout', help='set the SSH connection timeout in seconds'),
-        '-p': dict(long='--port', default=constants.DEFAULT_REMOTE_PORT, type='int',
-                   dest='remote_port', help='use this remote SSH port'),
-    }
+    parser = SortedOptParser(usage)
+    parser.add_option('-D','--debug', default=False, action="store_true",
+        help='enable standard error debugging of modules.')
+    parser.add_option('-f','--forks', dest='forks', default=constants.DEFAULT_FORKS, type='int',
+        help='number of parallel processes to use')
+    parser.add_option('-i', '--inventory-file', dest='inventory',
+       help='inventory host file', default=constants.DEFAULT_HOST_LIST)
+    parser.add_option('-k', '--ask-pass', default=False, action='store_true',
+        help='ask for SSH password')
+    parser.add_option('-M', '--module-path', dest='module_path',
+       help="path to module library", default=constants.DEFAULT_MODULE_PATH)
+    parser.add_option('-T', '--timeout', default=constants.DEFAULT_TIMEOUT, type='int',
+        dest='timeout', help='set the SSH timeout in seconds')
+    parser.add_option('-p', '--port', default=constants.DEFAULT_REMOTE_PORT, type='int',
+        dest='remote_port', help='set the remote ssh port')
 
     if output_opts:
-        options.update({
-            '-o' : dict(long='--one-line', dest='one_line', action='store_true',
-                        help='condense output'),
-            '-t' : dict(long='--tree', dest='tree', default=None,
-                        help='log results to this directory')
-        })
+        parser.add_option('-o', '--one-line', dest='one_line', action='store_true',
+            help='condense output')
+        parser.add_option('-t', '--tree', dest='tree', default=None,
+            help='log output to this directory')
 
     if runas_opts:
-        options.update({
-            '-s' : dict(long="--sudo", default=False, action="store_true",
-                        dest='sudo', help="run operations with sudo (nopasswd)"),
-            '-u' : dict(long='--user', default=constants.DEFAULT_REMOTE_USER,
-                        dest='remote_user', help='connect as this user'),
-        })
+        parser.add_option("-s", "--sudo", default=False, action="store_true",
+            dest='sudo', help="run operations with sudo (nopasswd)")
+        parser.add_option('-u', '--user', default=constants.DEFAULT_REMOTE_USER,
+            dest='remote_user', help='connect as this user')
     
     if connect_opts:
-        options.update({
-            '-c' : dict(long='--connection', dest='connection',
-                        choices=C.DEFAULT_TRANSPORT_OPTS,
-                        default=C.DEFAULT_TRANSPORT,
-                        help="connection type to use")
-        })
+        parser.add_option('-c', '--connection', dest='connection',
+                          choices=C.DEFAULT_TRANSPORT_OPTS,
+                          default=C.DEFAULT_TRANSPORT,
+                          help="connection type to use")
 
     if async_opts:
-        options.update({
-            '-P' : dict(long='--poll', default=constants.DEFAULT_POLL_INTERVAL, type='int',
-                        dest='poll_interval', help='set the poll interval if using -B'),
-            '-B' : dict(long='--background', dest='seconds', type='int', default=0,
-                        help='run asynchronously, failing after X seconds'),
-        })
+        parser.add_option('-P', '--poll', default=constants.DEFAULT_POLL_INTERVAL, type='int',
+            dest='poll_interval', help='set the poll interval if using -B')
+        parser.add_option('-B', '--background', dest='seconds', type='int', default=0,
+            help='run asynchronously, failing after X seconds')
 
-    return options
+    return parser