internet/community.general
3
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2026-05-07 13:52:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert templating enhancements from 73dbab70 e6c28658 d409352c 9858b1f2 4587528b 9b1fe455 214b0b05 8d3db803 7f9504d1 5031104c 35cb9dc2 2bd8cb57 1e85c754

Browse Source
This commit is contained in:
James Tanner
2013-10-30 10:50:16 -04:00
parent e0fbaea566
commit d154bf8781
11 changed files with 1244 additions and 1397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
lib/ansible/utils/__init__.py
View File

@@ -23,6 +23,12 @@ import yaml
import copy
import optparse
import operator
from ansible import errors
from ansible import __version__
from ansible.utils.plugins import *
from ansible.utils import template
from ansible.callbacks import display
import ansible.constants as C
import time
import StringIO
import stat
@@ -37,15 +43,6 @@ import getpass
import sys
import textwrap
# ansible
from ansible import errors
from ansible import __version__
from ansible.utils.plugins import *
from ansible.utils import template
from ansible.callbacks import display
import ansible.constants as C
import filesystem
VERBOSITY=0
# list of all deprecation messages to prevent duplicate display
@@ -227,7 +224,7 @@ def unfrackpath(path):
example:
'$HOME/../../var/mail' becomes '/var/spool/mail'
'''
return filesystem.unfrackpath(path)
return os.path.normpath(os.path.realpath(os.path.expandvars(os.path.expanduser(path))))
def prepare_writeable_dir(tree,mode=0777):
''' make sure a directory exists and is writeable '''
@@ -253,11 +250,33 @@ def path_dwim(basedir, given):
'''
make relative paths work like folks expect.
'''
return filesystem.path_dwim(basedir, given)
if given.startswith("/"):
return os.path.abspath(given)
elif given.startswith("~"):
return os.path.abspath(os.path.expanduser(given))
else:
return os.path.abspath(os.path.join(basedir, given))
def path_dwim_relative(original, dirname, source, playbook_base, check=True):
''' find one file in a directory one level up in a dir named dirname relative to current '''
return filesystem.path_dwim_relative(original, dirname, source, playbook_base, check=True)
# (used by roles code)
basedir = os.path.dirname(original)
if os.path.islink(basedir):
basedir = unfrackpath(basedir)
template2 = os.path.join(basedir, dirname, source)
else:
template2 = os.path.join(basedir, '..', dirname, source)
source2 = path_dwim(basedir, template2)
if os.path.exists(source2):
return source2
obvious_local_path = path_dwim(playbook_base, source)
if os.path.exists(obvious_local_path):
return obvious_local_path
if check:
raise errors.AnsibleError("input file not found at %s or %s" % (source2, obvious_local_path))
return source2 # which does not exist
def json_loads(data):
''' parse a JSON string and return a data structure '''

64
lib/ansible/utils/filesystem.py
View File

@@ -1,64 +0,0 @@
# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import sys
import os
from ansible import errors
from ansible import __version__
import ansible.constants as C
def path_dwim(basedir, given):
'''
make relative paths work like folks expect.
'''
if given.startswith("/"):
return os.path.abspath(given)
elif given.startswith("~"):
return os.path.abspath(os.path.expanduser(given))
else:
return os.path.abspath(os.path.join(basedir, given))
def path_dwim_relative(original, dirname, source, playbook_base, check=True):
''' find one file in a directory one level up in a dir named dirname relative to current '''
# (used by roles code)
basedir = os.path.dirname(original)
if os.path.islink(basedir):
basedir = unfrackpath(basedir)
template2 = os.path.join(basedir, dirname, source)
else:
template2 = os.path.join(basedir, '..', dirname, source)
source2 = path_dwim(basedir, template2)
if os.path.exists(source2):
return source2
obvious_local_path = path_dwim(playbook_base, source)
if os.path.exists(obvious_local_path):
return obvious_local_path
if check:
raise errors.AnsibleError("input file not found at %s or %s" % (source2, obvious_local_path))
return source2 # which does not exist
def unfrackpath(path):
'''
returns a path that is free of symlinks, environment
variables, relative path traversals and symbols (~)
example:
'$HOME/../../var/mail' becomes '/var/spool/mail'
'''
return os.path.normpath(os.path.realpath(os.path.expandvars(os.path.expanduser(path))))

345
lib/ansible/utils/template.py
View File

@@ -21,18 +21,15 @@ import codecs
import jinja2
from jinja2.runtime import StrictUndefined
from jinja2.exceptions import TemplateSyntaxError
import yaml
import json
from ansible import errors
import ansible.constants as C
import time
import subprocess
import datetime
import pwd
import ast
import plugins
import filesystem
FILTER_PLUGINS = None
_LISTRE = re.compile(r"(\w+)\[(\d+)\]")
JINJA2_OVERRIDE='#jinja2:'
class Globals(object):
@@ -47,9 +44,10 @@ def _get_filters():
if Globals.FILTERS is not None:
return Globals.FILTERS
my_plugins = [ x for x in plugins.filter_loader.all()]
from ansible import utils
plugins = [ x for x in utils.plugins.filter_loader.all()]
filters = {}
for fp in my_plugins:
for fp in plugins:
filters.update(fp.filters())
Globals.FILTERS = filters
@@ -57,17 +55,33 @@ def _get_filters():
def _get_extensions():
''' return jinja2 extensions to load '''
# if some extensions are set via jinja_extensions in ansible.cfg, we try
# to load them with the jinja environment
'''
if some extensions are set via jinja_extensions in ansible.cfg, we try
to load them with the jinja environment
'''
jinja_exts = []
if C.DEFAULT_JINJA2_EXTENSIONS:
# Let's make sure the configuration directive doesn't contain spaces
# and split extensions in an array
'''
Let's make sure the configuration directive doesn't contain spaces
and split extensions in an array
'''
jinja_exts = C.DEFAULT_JINJA2_EXTENSIONS.replace(" ", "").split(',')
return jinja_exts
class Flags:
LEGACY_TEMPLATE_WARNING = False
# TODO: refactor this file
FILTER_PLUGINS = None
_LISTRE = re.compile(r"(\w+)\[(\d+)\]")
JINJA2_OVERRIDE='#jinja2:'
def lookup(name, *args, **kwargs):
instance = plugins.lookup_loader.get(name.lower(), basedir=kwargs.get('basedir',None))
from ansible import utils
instance = utils.plugins.lookup_loader.get(name.lower(), basedir=kwargs.get('basedir',None))
vars = kwargs.get('vars', None)
if instance is not None:
@@ -81,15 +95,8 @@ def _legacy_varFindLimitSpace(basedir, vars, space, part, lookup_fatal, depth, e
basically does space.get(part, None), but with
templating for part and a few more things
DEPRECATED
LEGACY VARIABLES ARE SLATED FOR REMOVAL IN ANSIBLE 1.5
use {{ foo }} INSTEAD
'''
if not C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES:
raise Exception("we should not be here")
# Previous part couldn't be found, nothing to limit to
if space is None:
return space
@@ -140,16 +147,12 @@ def _legacy_varFind(basedir, text, vars, lookup_fatal, depth, expand_lists):
end=<index into text where the variable ends>)
or None if no variable could be found in text. If replacement is None, it should be replaced with the
original data in the caller.
DEPRECATED
LEGACY VARIABLES ARE SLATED FOR REMOVAL IN ANSIBLE 1.5
use {{ foo }} INSTEAD
'''
# short circuit this whole function if we have specified we don't want
# legacy var replacement
if not C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES:
raise Exception("we should not be here")
if C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES == False:
return None
start = text.find("$")
if start == -1:
@@ -221,6 +224,7 @@ def _legacy_varFind(basedir, text, vars, lookup_fatal, depth, expand_lists):
if basedir is None:
return {'replacement': None, 'start': start, 'end': end}
var_end -= 1
from ansible import utils
args = text[part_start:var_end]
if lookup_plugin_name == 'LOOKUP':
lookup_plugin_name, args = args.split(",", 1)
@@ -232,7 +236,7 @@ def _legacy_varFind(basedir, text, vars, lookup_fatal, depth, expand_lists):
return None
instance = plugins.lookup_loader.get(lookup_plugin_name.lower(), basedir=basedir)
instance = utils.plugins.lookup_loader.get(lookup_plugin_name.lower(), basedir=basedir)
if instance is not None:
try:
replacement = instance.run(args, inject=vars)
@@ -256,25 +260,16 @@ def _legacy_varFind(basedir, text, vars, lookup_fatal, depth, expand_lists):
return dict(replacement=space, start=start, end=end)
def legacy_varReplace(basedir, raw, vars, lookup_fatal=True, depth=0, expand_lists=False):
''' Perform variable replacement of $variables in string raw using vars dictionary
DEPRECATED
LEGACY VARIABLES ARE SLATED FOR REMOVAL IN ANSIBLE 1.5
use {{ foo }} INSTEAD
'''
if not C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES:
raise Exception("we should not be here")
# this code originally from yum (and later modified a lot)
''' Perform variable replacement of $variables in string raw using vars dictionary '''
# this code originally from yum
orig = raw
if not isinstance(raw, unicode):
raw = raw.decode("utf-8")
#if (depth > 20):
# raise errors.AnsibleError("template recursion depth exceeded")
if (depth > 20):
raise errors.AnsibleError("template recursion depth exceeded")
done = [] # Completed chunks to return
@@ -302,64 +297,12 @@ def legacy_varReplace(basedir, raw, vars, lookup_fatal=True, depth=0, expand_lis
result = ''.join(done)
previous_old_style_vars = orig.count('$')
new_old_style_vars = result.count('$')
if previous_old_style_vars != new_old_style_vars:
if result != orig:
from ansible import utils
utils.deprecated("Legacy variable substitution, such as using ${foo} or $foo instead of {{ foo }} is currently valid but will be phased out and has been out of favor since version 1.2. This is the last of legacy features on our deprecation list. You may continue to use this if you have specific needs for now","1.5")
utils.deprecated("Legacy variable subsitution, such as using ${foo} or $foo instead of {{ foo }} is currently valid but will be phased out and has been out of favor since version 1.2. This is the last of legacy features on our deprecation list. You may continue to use this if you have specific needs for now","1.6")
return result
def fix_ds(basedir, vars, original, depth=0):
''' used to massage the input dictionary to avoid surprises later and minimize more complex recursive problems '''
while (depth < 20):
depth = depth + 1
vars2 = _fix_ds(basedir, vars, original, depth=depth)
if vars2 == vars:
return vars
vars = vars2
return vars
def _fix_ds(basedir, vars, original, depth=0):
if isinstance(vars, dict):
return dict([ (k, fix_ds(basedir, v, original, depth=depth+1)) for (k,v) in vars.iteritems() ])
if isinstance(vars, (dict, tuple)):
return [ fix_ds(basedir, x,original, depth=depth+1) for x in vars ]
if isinstance(vars, basestring) and "{{" in vars and not "|" in vars and not "lookup(" in vars:
return lightweight_var_template(basedir, vars, original)
return vars
def lightweight_var_template(basedir, input, vars):
return template_from_string(basedir, input, vars, fail_on_undefined=False, lookups=True, filters=True)
def template(basedir, input_value, vars, lookup_fatal=True, depth=-1, expand_lists=True, convert_bare=False, fail_on_undefined=False, filter_fatal=True, lookups=True):
vars = fix_ds(basedir, vars, vars.copy())
last_time = input_value
result = None
changed = True
while changed:
result = _template(
basedir,
last_time,
vars,
lookup_fatal=lookup_fatal,
depth=depth,
expand_lists=expand_lists,
convert_bare=convert_bare,
fail_on_undefined=fail_on_undefined,
filter_fatal=filter_fatal,
lookups=lookups,
)
if last_time == result:
changed = False
last_time = result
depth = depth + 1
if depth > 20:
raise errors.AnsibleError("template recursion depth exceeded")
return result
def _template(basedir, varname, vars, lookup_fatal=True, depth=0, expand_lists=True, convert_bare=False, fail_on_undefined=False, filter_fatal=True, lookups=True):
def template(basedir, varname, vars, lookup_fatal=True, depth=0, expand_lists=True, convert_bare=False, fail_on_undefined=False, filter_fatal=True):
''' templates a data structure by traversing it and substituting for other data structures '''
try:
@@ -370,47 +313,109 @@ def _template(basedir, varname, vars, lookup_fatal=True, depth=0, expand_lists=T
if isinstance(varname, basestring):
if '{{' in varname or '{%' in varname:
varname = template_from_string(basedir, varname, vars, fail_on_undefined, lookups=lookups)
if not C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES:
return varname
varname = template_from_string(basedir, varname, vars, fail_on_undefined)
if not '$' in varname:
return varname
m = _legacy_varFind(basedir, varname, vars, lookup_fatal, depth, expand_lists)
if not m:
return varname
if m['start'] == 0 and m['end'] == len(varname):
if m['replacement'] is not None:
Flags.LEGACY_TEMPLATE_WARNING = True
return template(basedir, m['replacement'], vars, lookup_fatal, depth, expand_lists)
else:
return varname
else:
Flags.LEGACY_TEMPLATE_WARNING = True
return legacy_varReplace(basedir, varname, vars, lookup_fatal, depth, expand_lists)
elif isinstance(varname, (list, tuple)):
return [ template(basedir, v, vars, lookup_fatal, depth, expand_lists, fail_on_undefined=fail_on_undefined) for v in varname]
return [template(basedir, v, vars, lookup_fatal, depth, expand_lists, fail_on_undefined=fail_on_undefined) for v in varname]
elif isinstance(varname, dict):
return dict([
(k, template(
basedir, v, vars, lookup_fatal, depth, expand_lists, fail_on_undefined=fail_on_undefined)
) for (k,v) in varname.iteritems()
])
d = {}
for (k, v) in varname.iteritems():
d[k] = template(basedir, v, vars, lookup_fatal, depth, expand_lists, fail_on_undefined=fail_on_undefined)
return d
else:
return varname
except errors.AnsibleFilterError:
if filter_fatal:
raise
else:
return varname
class _jinja2_vars(object):
'''
Helper class to template all variable content before jinja2 sees it.
This is done by hijacking the variable storage that jinja2 uses, and
overriding __contains__ and __getitem__ to look like a dict. Added bonus
is avoiding duplicating the large hashes that inject tends to be.
To facilitate using builtin jinja2 things like range, globals are handled
here.
extras is a list of locals to also search for variables.
'''
def __init__(self, basedir, vars, globals, fail_on_undefined, *extras):
self.basedir = basedir
self.vars = vars
self.globals = globals
self.fail_on_undefined = fail_on_undefined
self.extras = extras
def __contains__(self, k):
if k in self.vars:
return True
for i in self.extras:
if k in i:
return True
if k in self.globals:
return True
return False
def __getitem__(self, varname):
if varname not in self.vars:
for i in self.extras:
if varname in i:
return i[varname]
if varname in self.globals:
return self.globals[varname]
else:
raise KeyError("undefined variable: %s" % varname)
var = self.vars[varname]
# HostVars is special, return it as-is
if isinstance(var, dict) and type(var) != dict:
return var
else:
return template(self.basedir, var, self.vars, fail_on_undefined=self.fail_on_undefined)
def add_locals(self, locals):
'''
If locals are provided, create a copy of self containing those
locals in addition to what is already in this variable proxy.
'''
if locals is None:
return self
return _jinja2_vars(self.basedir, self.vars, self.globals, self.fail_on_undefined, locals, *self.extras)
class J2Template(jinja2.environment.Template):
'''
This class prevents Jinja2 from running _jinja2_vars through dict()
Without this, {% include %} and similar will create new contexts unlike
the special one created in template_from_file. This ensures they are all
alike, with the exception of potential locals.
'''
def new_context(self, vars=None, shared=False, locals=None):
return jinja2.runtime.Context(self.environment, vars.add_locals(locals), self.name, self.blocks)
def template_from_file(basedir, path, vars):
''' run a file through the templating engine '''
fail_on_undefined = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
realpath = filesystem.path_dwim(basedir, path)
from ansible import utils
realpath = utils.path_dwim(basedir, path)
loader=jinja2.FileSystemLoader([basedir,os.path.dirname(realpath)])
def my_lookup(*args, **kwargs):
@@ -440,20 +445,27 @@ def template_from_file(basedir, path, vars):
(key,val) = pair.split(':')
setattr(environment,key.strip(),ast.literal_eval(val.strip()))
environment.template_class = J2Template
try:
t = environment.from_string(data)
except TemplateSyntaxError, e:
# Throw an exception which includes a more user friendly error message
values = {'name': realpath, 'lineno': e.lineno, 'error': str(e)}
msg = 'file: %(name)s, line number: %(lineno)s, error: %(error)s' % \
values
error = errors.AnsibleError(msg)
raise error
vars = vars.copy()
try:
template_uid = pwd.getpwuid(os.stat(realpath).st_uid).pw_name
except:
template_uid = os.stat(realpath).st_uid
vars.update(dict(
template_host = os.uname()[1],
template_path = realpath,
template_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(realpath)),
template_uid = template_uid,
template_fullpath = os.path.abspath(realpath),
template_run_date = datetime.datetime.now(),
))
vars['template_host'] = os.uname()[1]
vars['template_path'] = realpath
vars['template_mtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(realpath))
vars['template_uid'] = template_uid
vars['template_fullpath'] = os.path.abspath(realpath)
vars['template_run_date'] = datetime.datetime.now()
managed_default = C.DEFAULT_MANAGED_STR
managed_str = managed_default.format(
@@ -466,75 +478,60 @@ def template_from_file(basedir, path, vars):
time.localtime(os.path.getmtime(realpath))
)
# this double template pass is here to detect errors while we still have context
# actual recursion is handled by the mainline template function further down
# This line performs deep Jinja2 magic that uses the _jinja2_vars object for vars
# Ideally, this could use some API where setting shared=True and the object won't get
# passed through dict(o), but I have not found that yet.
try:
t = environment.from_string(data)
res = t.render(vars)
res = jinja2.utils.concat(t.root_render_func(t.new_context(_jinja2_vars(basedir, vars, t.globals, fail_on_undefined), shared=True)))
except jinja2.exceptions.UndefinedError, e:
raise errors.AnsibleUndefinedVariable("One or more undefined variables: %s" % str(e))
except TemplateSyntaxError, e:
# Throw an exception which includes a more user friendly error message
values = dict(name=realpath, lineno=e.lineno, error=str(e))
msg = 'file: %(name)s, line number: %(lineno)s, error: %(error)s' % values
error = errors.AnsibleError(msg)
raise error
if data.endswith('\n') and not res.endswith('\n'):
res = res + '\n'
return template(basedir, res, vars)
return template(basedir, res, vars)
def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=True, filters=True):
def template_from_string(basedir, data, vars, fail_on_undefined=False):
''' run a string through the (Jinja2) templating engine '''
def my_lookup(*args, **kwargs):
kwargs['vars'] = vars
return lookup(*args, basedir=basedir, **kwargs)
if type(data) == str:
data = unicode(data, 'utf-8')
environment = jinja2.Environment(trim_blocks=True, undefined=StrictUndefined, extensions=_get_extensions())
if filters:
environment.filters.update(_get_filters())
if '_original_file' in vars:
basedir = os.path.dirname(vars['_original_file'])
filesdir = os.path.abspath(os.path.join(basedir, '..', 'files'))
if os.path.exists(filesdir):
basedir = filesdir
# TODO: may need some way of using lookup plugins here seeing we aren't calling
# the legacy engine, lookup() as a function, perhaps?
if type(data) == str:
data = unicode(data, 'utf-8')
environment = jinja2.Environment(trim_blocks=True, undefined=StrictUndefined, extensions=_get_extensions())
environment.filters.update(_get_filters())
if '_original_file' in vars:
basedir = os.path.dirname(vars['_original_file'])
filesdir = os.path.abspath(os.path.join(basedir, '..', 'files'))
if os.path.exists(filesdir):
basedir = filesdir
# TODO: may need some way of using lookup plugins here seeing we aren't calling
# the legacy engine, lookup() as a function, perhaps?
try:
t = environment.from_string(data.decode('utf-8'))
except Exception, e:
if 'recursion' in str(e):
raise errors.AnsibleError("recursive loop detected in template string: %s" % data)
else:
return data
if type(data) == str:
data = unicode(data, 'utf-8')
environment = jinja2.Environment(trim_blocks=True, undefined=StrictUndefined, extensions=_get_extensions())
environment.filters.update(_get_filters())
environment.template_class = J2Template
if '_original_file' in vars:
basedir = os.path.dirname(vars['_original_file'])
filesdir = os.path.abspath(os.path.join(basedir, '..', 'files'))
if os.path.exists(filesdir):
basedir = filesdir
# TODO: may need some way of using lookup plugins here seeing we aren't calling
# the legacy engine, lookup() as a function, perhaps?
data = data.decode('utf-8')
try:
t = environment.from_string(data)
except Exception, e:
if 'recursion' in str(e):
raise errors.AnsibleError("recursive loop detected in template string: %s" % data)
else:
return data
def my_lookup(*args, **kwargs):
kwargs['vars'] = vars
return lookup(*args, basedir=basedir, **kwargs)
if lookups:
t.globals['lookup'] = my_lookup
try:
return t.render(vars)
except jinja2.exceptions.UndefinedError:
jvars =_jinja2_vars(basedir, vars, t.globals, fail_on_undefined)
new_context = t.new_context(jvars, shared=True)
rf = t.root_render_func(new_context)
res = jinja2.utils.concat(rf)
return res
except (jinja2.exceptions.UndefinedError, errors.AnsibleUndefinedVariable):
if fail_on_undefined:
raise
else: