Ziploader

* Ziploader proof of concept (jimi-c)

* Cleanups to proof of concept ziploader branch:

* python3 compatible base64 encoding
* zipfile compression (still need to enable toggling this off for
  systems without zlib support in python)
* Allow non-wildcard imports (still need to make this recusrsive so that
  we can have module_utils code that imports other module_utils code.)
* Better tracebacks: module filename is kept and module_utils directory
  is kept so that tracebacks show the real filenames that the errors
  appear in.

* Make sure we import modules that are used into the module_utils files that they are used in.

* Set ansible version in a more pythonic way for ziploader than we were doing in module replacer

* Make it possible to set the module compression as an inventory var

This may be necessary on systems where python has been compiled without
zlib compression.

* Refactoring of module_common code:

* module replacer only replaces values that make sense for that type of
  file (example: don't attempt to replace python imports if we're in
  a powershell module).
* Implement configurable shebang support for ziploader wrapper
* Implement client-side constants (for SELINUX_SPECIAL_FS and SYSLOG)
  via environment variable.
* Remove strip_comments param as we're never going to use it (ruins line
  numbering)

* Don't repeat ourselves about detecting REPLACER

* Add an easy way to debug

* Port test-module to the ziploader-aware modify_module()

* strip comments and blank lines from the wrapper so we send less over the wire.

* Comments cleanup

* Remember to output write the module line itself in powershell modules

* for line in lines strips the newlines so we have to add them back in

lib/ansible/module_utils/a10.py

@@ -27,8 +27,8 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import json
-# Note: modules using this must have from ansible.module_utils.urls import *
-# before this is imported
+
+from ansible.module_utils.urls import fetch_url
 
 AXAPI_PORT_PROTOCOLS = {
     'tcp': 2,

lib/ansible/module_utils/basic.py

@@ -27,25 +27,13 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-# == BEGIN DYNAMICALLY INSERTED CODE ==
-
-ANSIBLE_VERSION = "<<ANSIBLE_VERSION>>"
-
-MODULE_ARGS = "<<INCLUDE_ANSIBLE_MODULE_ARGS>>"
-MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
-
 BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1, True]
 BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0, False]
 BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE
 
-SELINUX_SPECIAL_FS="<<SELINUX_SPECIAL_FILESYSTEMS>>"
-
 # ansible modules can be written in any language.  To simplify
-# development of Python modules, the functions available here
-# can be inserted in any module source automatically by including
-# #<<INCLUDE_ANSIBLE_MODULE_COMMON>> on a blank line by itself inside
-# of an ansible module. The source of this common code lives
-# in ansible/executor/module_common.py
+# development of Python modules, the functions available here can
+# be used to do many common tasks
 
 import locale
 import os
@@ -231,6 +219,27 @@ except ImportError:
 
 _literal_eval = literal_eval
 
+from ansible import __version__
+# Backwards compat. New code should just import and use __version__
+ANSIBLE_VERSION = __version__
+
+try:
+    # MODULE_COMPLEX_ARGS is an old name kept for backwards compat
+    MODULE_COMPLEX_ARGS = os.environ.pop('ANSIBLE_MODULE_ARGS')
+except KeyError:
+    # This file might be used for its utility functions.  So don't fail if
+    # running outside of a module environment (will fail in _load_params()
+    # instead)
+    MODULE_COMPLEX_ARGS = None
+
+try:
+    # ARGS are for parameters given in the playbook.  Constants are for things
+    # that ansible needs to configure controller side but are passed to all
+    # modules.
+    MODULE_CONSTANTS = os.environ.pop('ANSIBLE_MODULE_CONSTANTS')
+except KeyError:
+    MODULE_CONSTANTS = None
+
 FILE_COMMON_ARGUMENTS=dict(
     src = dict(),
     mode = dict(type='raw'),
@@ -539,7 +548,8 @@ class AnsibleModule(object):
                 if k not in self.argument_spec:
                     self.argument_spec[k] = v
 
-        self.params = self._load_params()
+        self._load_constants()
+        self._load_params()
 
         # append to legal_inputs and then possibly check against them
         try:
@@ -754,7 +764,7 @@ class AnsibleModule(object):
             (device, mount_point, fstype, options, rest) = line.split(' ', 4)
 
             if path_mount_point == mount_point:
-                for fs in SELINUX_SPECIAL_FS.split(','):
+                for fs in self.constants['SELINUX_SPECIAL_FS']:
                     if fs in fstype:
                         special_context = self.selinux_context(path_mount_point)
                         return (True, special_context)
@@ -1412,16 +1422,38 @@ class AnsibleModule(object):
                     self.params[k] = default
 
     def _load_params(self):
-        ''' read the input and return a dictionary and the arguments string '''
+        ''' read the input and set the params attribute'''
+        if MODULE_COMPLEX_ARGS is None:
+            # This helper used too early for fail_json to work.
+            print('{"msg": "Error: ANSIBLE_MODULE_ARGS not found in environment.  Unable to figure out what parameters were passed", "failed": true}')
+            sys.exit(1)
+
         params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS))
         if params is None:
             params = dict()
-        return params
+        self.params = params
+
+    def _load_constants(self):
+        ''' read the input and set the constants attribute'''
+        if MODULE_CONSTANTS is None:
+            # This helper used too early for fail_json to work.
+            print('{"msg": "Error: ANSIBLE_MODULE_CONSTANTS not found in environment.  Unable to figure out what constants were passed", "failed": true}')
+            sys.exit(1)
+
+        # Make constants into "native string"
+        if sys.version_info >= (3,):
+            constants = json_dict_bytes_to_unicode(json.loads(MODULE_CONSTANTS))
+        else:
+            constants = json_dict_unicode_to_bytes(json.loads(MODULE_CONSTANTS))
+        if constants is None:
+            constants = dict()
+        self.constants = constants
 
     def _log_to_syslog(self, msg):
         if HAS_SYSLOG:
             module = 'ansible-%s' % os.path.basename(__file__)
-            syslog.openlog(str(module), 0, syslog.LOG_USER)
+            facility = getattr(syslog, self.constants.get('SYSLOG_FACILITY', 'LOG_USER'), syslog.LOG_USER)
+            syslog.openlog(str(module), 0, facility)
             syslog.syslog(syslog.LOG_INFO, msg)
 
     def debug(self, msg):

lib/ansible/module_utils/ec2.py

@@ -25,9 +25,16 @@
 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 import os
 from time import sleep
 
+try:
+    import boto
+    HAS_BOTO = True
+except ImportError:
+    HAS_BOTO = False
+
 try:
     import boto3
     import botocore

lib/ansible/module_utils/eos.py

@@ -18,6 +18,13 @@
 #
 import os
 
+import re
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.shell import Shell, Command, HAS_PARAMIKO
+from ansible.module_utils.netcfg import parse
+from ansible.module_utils.urls import fetch_url
+
 NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
 
 NET_COMMON_ARGS = dict(

lib/ansible/module_utils/gce.py

@@ -29,6 +29,8 @@
 
 import os
 import traceback
+
+from libcloud.compute.types import Provider
 from libcloud.compute.providers import get_driver
 
 USER_AGENT_PRODUCT="Ansible-gce"

lib/ansible/module_utils/ios.py

@@ -17,6 +17,12 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import re
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.shell import Shell, Command, HAS_PARAMIKO
+from ansible.module_utils.netcfg import parse
+
 NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
 
 NET_COMMON_ARGS = dict(

lib/ansible/module_utils/iosxr.py

@@ -17,6 +17,12 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import re
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.shell import Shell, HAS_PARAMIKO
+from ansible.module_utils.netcfg import parse
+
 NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
 
 NET_COMMON_ARGS = dict(

lib/ansible/module_utils/junos.py

@@ -17,6 +17,10 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.shell import Shell, HAS_PARAMIKO
+from ansible.module_utils.netcfg import parse
+
 NET_COMMON_ARGS = dict(
     host=dict(required=True),
     port=dict(default=22, type='int'),

lib/ansible/module_utils/mysql.py

@@ -27,7 +27,13 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import os
 
+try:
+    import MySQLdb
+    mysqldb_found = True
+except ImportError:
+    mysqldb_found = False
 
 def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None, connect_timeout=30):
     config = {

lib/ansible/module_utils/netcfg.py

@@ -20,6 +20,9 @@
 import re
 import collections
 import itertools
+import shlex
+
+from ansible.module_utils.basic import BOOLEANS_TRUE, BOOLEANS_FALSE
 
 DEFAULT_COMMENT_TOKENS = ['#', '!']
 

lib/ansible/module_utils/nxos.py

@@ -16,6 +16,14 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
+
+import re
+
+from ansible.module_utils.urls import fetch_url
+from ansible.module_utils.shell import Shell, HAS_PARAMIKO
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.netcfg import parse
+
 NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
 
 NET_COMMON_ARGS = dict(
@@ -117,8 +125,6 @@ class Nxapi(object):
                     (command_type, ','.join(NXAPI_COMMAND_TYPES))
             self.module_fail_json(msg=msg)
 
-        debug = dict()
-
         data = self._get_body(clist, command_type, encoding)
         data = self.module.jsonify(data)
 

lib/ansible/module_utils/openswitch.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
+import re
 import time
 import json
 
@@ -28,6 +29,11 @@ try:
 except ImportError:
     HAS_OPS = False
 
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.urls import fetch_url
+from ansible.module_utils.shell import Shell, HAS_PARAMIKO
+from ansible.module_utils.netcfg import parse
+
 NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
 
 NET_COMMON_ARGS = dict(

lib/ansible/module_utils/powershell.ps1

@@ -32,7 +32,7 @@ Set-StrictMode -Version 2.0
 # JSON; assign them to an environment variable and redefine $args so existing
 # modules will continue to work.
 $complex_args = @'
-<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>
+<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>
 '@
 Set-Content env:MODULE_COMPLEX_ARGS -Value $complex_args
 $args = @('env:MODULE_COMPLEX_ARGS')

lib/ansible/module_utils/rax.py

@@ -32,6 +32,8 @@ import os
 import re
 from uuid import UUID
 
+from ansible import __version__
+from ansible.module_utils.basic import BOOLEANS
 
 FINAL_STATUSES = ('ACTIVE', 'ERROR')
 VOLUME_STATUS = ('available', 'attaching', 'creating', 'deleting', 'in-use',
@@ -262,7 +264,7 @@ def rax_required_together():
 
 def setup_rax_module(module, rax_module, region_required=True):
     """Set up pyrax in a standard way for all modules"""
-    rax_module.USER_AGENT = 'ansible/%s %s' % (ANSIBLE_VERSION,
+    rax_module.USER_AGENT = 'ansible/%s %s' % (__version__,
                                                rax_module.USER_AGENT)
 
     api_key = module.params.get('api_key')

lib/ansible/module_utils/shell.py

@@ -146,7 +146,7 @@ class Shell(object):
                 cmd = '%s\r' % str(command)
                 self.shell.sendall(cmd)
                 responses.append(self.receive(command))
-        except socket.timeout, exc:
+        except socket.timeout:
             raise ShellError("timeout trying to send command", cmd)
         return responses
 

lib/ansible/module_utils/urls.py

@@ -81,6 +81,18 @@
 # agrees to be bound by the terms and conditions of this License
 # Agreement.
 
+import httplib
+import netrc
+import os
+import re
+import sys
+import socket
+import platform
+import tempfile
+import base64
+
+from ansible.module_utils.basic import get_distribution
+
 try:
     import urllib2
     HAS_URLLIB2 = True
@@ -151,8 +163,6 @@ if not HAS_MATCH_HOSTNAME:
 
     """The match_hostname() function from Python 3.4, essential when using SSL."""
 
-    import re
-
     class CertificateError(ValueError):
         pass
 
@@ -257,17 +267,6 @@ if not HAS_MATCH_HOSTNAME:
     HAS_MATCH_HOSTNAME = True
 
 
-import httplib
-import netrc
-import os
-import re
-import sys
-import socket
-import platform
-import tempfile
-import base64
-
-
 # This is a dummy cacert provided for Mac OS since you need at least 1
 # ca cert, regardless of validity, for Python on Mac OS to use the
 # keychain functionality in OpenSSL for validating SSL certificates.

lib/ansible/module_utils/vca.py

@@ -21,6 +21,8 @@ try:
 except ImportError:
     HAS_PYVCLOUD = False
 
+from ansible.module_utils.basic import AnsibleModule
+
 SERVICE_MAP = {'vca': 'ondemand', 'vchs': 'subscription', 'vcd': 'vcd'}
 LOGIN_HOST = {'vca': 'vca.vmware.com', 'vchs': 'vchs.vmware.com'}
 

lib/ansible/module_utils/vmware.py

@@ -25,7 +25,7 @@ try:
     # requests is required for exception handling of the ConnectionError
     import requests
     from pyVim import connect
-    from pyVmomi import vim, vmodl
+    from pyVmomi import vim
     HAS_PYVMOMI = True
 except ImportError:
     HAS_PYVMOMI = False