From 0a39700b36788ae88279b65e441c15cf15ae15ea Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 25 Aug 2016 14:58:35 -0700 Subject: [PATCH 01/30] Fix octal output in a few more places (#17250) Fix filetree lookup plugin for python3 (octal output and selinux API takes native strings) --- docsite/rst/developing_modules_python3.rst | 23 ++++++++++++++++++++++ lib/ansible/module_utils/basic.py | 4 ++-- lib/ansible/plugins/lookup/filetree.py | 23 ++++++++-------------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docsite/rst/developing_modules_python3.rst b/docsite/rst/developing_modules_python3.rst index 612a4ef606..b4e7baeafb 100644 --- a/docsite/rst/developing_modules_python3.rst +++ b/docsite/rst/developing_modules_python3.rst @@ -131,6 +131,29 @@ modules should create their octals like this:: # Can't use 0755 on Python-3 and can't use 0o755 on Python-2.4 EXECUTABLE_PERMS = int('0755', 8) +Outputting octal numbers may also need to be changed. In python2 we often did +this to return file permissions:: + + mode = int('0775', 8) + result['mode'] = oct(mode) + +This would give the user ``result['mode'] == '0755'`` in their playbook. In +python3, :func:`oct` returns the format with the lowercase ``o`` in it like: +``result['mode'] == '0o755'``. If a user had a conditional in their playbook +or was using the mode in a template the new format might break things. We +need to return the old form of mode for backwards compatibility. You can do +it like this:: + + mode = int('0775', 8) + result['mode'] = '0%03o' % mode + +You should use this wherever backwards compatibility is a concern or you are +dealing with file permissions. (With file permissions a user may be feeding +the mode into another program or to another module which doesn't understand +the python syntax for octal numbers. ``[zero][digit][digit][digit]`` is +understood by most everything and therefore the right way to express octals in +these cisrcumstances. + Bundled six ----------- diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index e8014d1c33..62f9e8d7d7 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1013,10 +1013,10 @@ class AnsibleModule(object): if diff is not None: if 'before' not in diff: diff['before'] = {} - diff['before']['mode'] = oct(prev_mode) + diff['before']['mode'] = '0%03o' % prev_mode if 'after' not in diff: diff['after'] = {} - diff['after']['mode'] = oct(mode) + diff['after']['mode'] = '0%03o' % mode if self.check_mode: return True diff --git a/lib/ansible/plugins/lookup/filetree.py b/lib/ansible/plugins/lookup/filetree.py index d0cbe298fc..f4d96af876 100644 --- a/lib/ansible/plugins/lookup/filetree.py +++ b/lib/ansible/plugins/lookup/filetree.py @@ -23,6 +23,8 @@ import grp import stat from ansible.plugins.lookup import LookupBase +from ansible.utils.unicode import to_str + from __main__ import display warning = display.warning @@ -33,25 +35,15 @@ try: except ImportError: pass -def _to_filesystem_str(path): - '''Returns filesystem path as a str, if it wasn't already. - - Used in selinux interactions because it cannot accept unicode - instances, and specifying complex args in a playbook leaves - you with unicode instances. This method currently assumes - that your filesystem encoding is UTF-8. - - ''' - if isinstance(path, unicode): - path = path.encode("utf-8") - return path # If selinux fails to find a default, return an array of None def selinux_context(path): context = [None, None, None, None] if HAVE_SELINUX and selinux.is_selinux_enabled(): try: - ret = selinux.lgetfilecon_raw(_to_filesystem_str(path)) + # note: the selinux module uses byte strings on python2 and text + # strings on python3 + ret = selinux.lgetfilecon_raw(to_str(path)) except OSError: return context if ret[0] != -1: @@ -60,6 +52,7 @@ def selinux_context(path): context = ret[1].split(':', 3) return context + def file_props(root, path): ''' Returns dictionary with file properties, or return None on failure ''' abspath = os.path.join(root, path) @@ -94,7 +87,7 @@ def file_props(root, path): ret['group'] = grp.getgrgid(st.st_gid).gr_name except KeyError: ret['group'] = st.st_gid - ret['mode'] = str(oct(stat.S_IMODE(st.st_mode))) + ret['mode'] = '0%03o' % (stat.S_IMODE(st.st_mode)) ret['size'] = st.st_size ret['mtime'] = st.st_mtime ret['ctime'] = st.st_ctime @@ -129,4 +122,4 @@ class LookupModule(LookupBase): if props is not None: ret.append(props) - return ret \ No newline at end of file + return ret From e375bfd6a588c4b3044b2c2a26b9553f83f63002 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Thu, 25 Aug 2016 17:50:22 -0500 Subject: [PATCH 02/30] Use post_validated play for serial calculations in TQM Fixes #17185 --- lib/ansible/executor/task_queue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index 407f02f227..c3313ae50a 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -224,10 +224,10 @@ class TaskQueueManager: num_hosts = len(self._inventory.get_hosts(new_play.hosts)) max_serial = 0 - if play.serial: + if new_play.serial: # the play has not been post_validated here, so we may need # to convert the scalar value to a list at this point - serial_items = play.serial + serial_items = new_play.serial if not isinstance(serial_items, list): serial_items = [serial_items] max_serial = max([pct_to_int(x, num_hosts) for x in serial_items]) From 5036bba2e0e47c9dd07fdaf8e4c8fd41d541b014 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 25 Aug 2016 22:10:00 -0400 Subject: [PATCH 03/30] implements command_string property when preparing commands * commands that need | json added now use command_string property * adds additonal keyword args in exception handling for json commands --- lib/ansible/module_utils/nxos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/module_utils/nxos.py b/lib/ansible/module_utils/nxos.py index 32dfd9be26..0378b14954 100644 --- a/lib/ansible/module_utils/nxos.py +++ b/lib/ansible/module_utils/nxos.py @@ -251,7 +251,7 @@ class Cli(CliBase): except ValueError: raise NetworkError( msg='unable to load response from device', - response=responses[index] + response=responses[index], command=str(cmd) ) return responses @@ -287,5 +287,5 @@ def prepare_commands(commands): jsonify = lambda x: '%s | json' % x for cmd in to_list(commands): if cmd.output == 'json': - cmd.command = jsonify(cmd) + cmd.command_string = jsonify(cmd) yield cmd From 9f1ac47f70ba7b5d6172688386cd828639bb03e0 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 25 Aug 2016 22:08:11 -0400 Subject: [PATCH 04/30] adds new property command_string to Command object This adds a new property to the Command object that is used to hold modified command strings that could be different from the command used to create the object. This allows for seamless switch between text and json enabled commands. --- lib/ansible/module_utils/netcli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ansible/module_utils/netcli.py b/lib/ansible/module_utils/netcli.py index 90d8bafeef..1f47cb239d 100644 --- a/lib/ansible/module_utils/netcli.py +++ b/lib/ansible/module_utils/netcli.py @@ -102,6 +102,7 @@ class Command(object): self.command = command self.output = output + self.command_string = command self.prompt = prompt self.response = response @@ -110,7 +111,7 @@ class Command(object): self.delay = delay def __str__(self): - return self.command + return self.command_string class CommandRunner(object): @@ -145,7 +146,7 @@ class CommandRunner(object): return cmdobj.response except KeyError: for cmd in self.commands: - if str(cmd) == command and cmd.output == output: + if cmd.command == command and cmd.output == output: self._cache[(command, output)] = cmd return cmd.response raise ValueError("command '%s' not found" % command) From f57f33a8e73d1ac10b3eb6b4333e635c1608bc27 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 25 Aug 2016 21:45:38 -0700 Subject: [PATCH 05/30] Fix fetch idempotence (#17255) Fetch always follows symlinks when downloading so it needs to always follow symlinks when getting the checksum of the file as well. --- lib/ansible/plugins/action/fetch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ansible/plugins/action/fetch.py b/lib/ansible/plugins/action/fetch.py index 7b04b52690..693c8ddc8d 100644 --- a/lib/ansible/plugins/action/fetch.py +++ b/lib/ansible/plugins/action/fetch.py @@ -64,7 +64,8 @@ class ActionModule(ActionBase): remote_checksum = None if not self._play_context.become: # calculate checksum for the remote file, don't bother if using become as slurp will be used - remote_checksum = self._remote_checksum(source, all_vars=task_vars) + # Force remote_checksum to follow symlinks because fetch always follows symlinks + remote_checksum = self._remote_checksum(source, all_vars=task_vars, follow=True) # use slurp if permissions are lacking or privilege escalation is needed remote_data = None From 9ac20e231df51fc6898b53a3188a1580c3a38649 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 26 Aug 2016 01:07:00 -0700 Subject: [PATCH 06/30] Update core submodule to pull in lineinfile py3 fix --- lib/ansible/modules/core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index ef84dbbddd..4912ec30a7 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit ef84dbbddd5d64e0860bd1f198dbd71929061d01 +Subproject commit 4912ec30a71b09577a78f940c6a772b38b76f1a2 From 4a3a9c0f2d873096fca604a6d5733ad9d735f9b0 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 26 Aug 2016 01:30:46 -0700 Subject: [PATCH 07/30] Fix for run_command on py3 and enable lineinfile test on py3 (#17257) * run_command needed a bit of tweaking to its string handling of arguments. * The run_command change fixes the last bit of lineinfile so we can enable its tests --- lib/ansible/module_utils/basic.py | 18 +++++++++--------- .../shippable/python3-test-tag-blacklist.txt | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 62f9e8d7d7..883e488a3f 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -139,7 +139,7 @@ from ansible.module_utils.pycompat24 import get_exception, literal_eval from ansible.module_utils.six import (PY2, PY3, b, binary_type, integer_types, iteritems, text_type, string_types) from ansible.module_utils.six.moves import map, reduce -from ansible.module_utils._text import to_native +from ansible.module_utils._text import to_native, to_bytes, to_text _NUMBERTYPES = tuple(list(integer_types) + [float]) @@ -2039,13 +2039,13 @@ class AnsibleModule(object): shell = True elif isinstance(args, (binary_type, text_type)) and use_unsafe_shell: shell = True - elif isinstance(args, string_types): + elif isinstance(args, (binary_type, text_type)): # On python2.6 and below, shlex has problems with text type # On python3, shlex needs a text type. - if PY2 and isinstance(args, text_type): - args = args.encode('utf-8') - elif PY3 and isinstance(args, binary_type): - args = args.decode('utf-8', errors='surrogateescape') + if PY2: + args = to_bytes(args) + elif PY3: + args = to_text(args, errors='surrogateescape') args = shlex.split(args) else: msg = "Argument 'args' to run_command must be list or string" @@ -2055,9 +2055,9 @@ class AnsibleModule(object): if prompt_regex: if isinstance(prompt_regex, text_type): if PY3: - prompt_regex = prompt_regex.encode('utf-8', errors='surrogateescape') + prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') elif PY2: - prompt_regex = prompt_regex.encode('utf-8') + prompt_regex = to_bytes(prompt_regex) try: prompt_re = re.compile(prompt_regex, re.MULTILINE) except re.error: @@ -2065,7 +2065,7 @@ class AnsibleModule(object): # expand things like $HOME and ~ if not shell: - args = [ os.path.expandvars(os.path.expanduser(x)) for x in args if x is not None ] + args = [ os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None ] rc = 0 msg = None diff --git a/test/utils/shippable/python3-test-tag-blacklist.txt b/test/utils/shippable/python3-test-tag-blacklist.txt index 198b49a0f0..aa3b503f0f 100644 --- a/test/utils/shippable/python3-test-tag-blacklist.txt +++ b/test/utils/shippable/python3-test-tag-blacklist.txt @@ -9,7 +9,6 @@ test_get_url test_git test_hg test_iterators -test_lineinfile test_lookups test_mysql_db test_mysql_user From a942758a0714c6176cc09a78562e9f3c62b2c8ff Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Fri, 26 Aug 2016 14:53:28 +0200 Subject: [PATCH 08/30] Fix wrong error class (#17259) AnsibleError is not imported in that file, and since that's a parsing time issue, better raise AnsibleParserError like the rest of the file. Issue signaled on irc by gordon` --- lib/ansible/playbook/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/playbook/helpers.py b/lib/ansible/playbook/helpers.py index d1b2d0ed7a..fc6837dfbc 100644 --- a/lib/ansible/playbook/helpers.py +++ b/lib/ansible/playbook/helpers.py @@ -191,7 +191,7 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h if data is None: return [] elif not isinstance(data, list): - raise AnsibleError("included task files must contain a list of tasks", obj=data) + raise AnsibleParserError("included task files must contain a list of tasks", obj=data) # since we can't send callbacks here, we display a message directly in # the same fashion used by the on_include callback. We also do it here, From 238cccf1668579a12c3b80ef9baa27f432accb65 Mon Sep 17 00:00:00 2001 From: Alexander Stock Date: Fri, 26 Aug 2016 16:41:17 +0200 Subject: [PATCH 09/30] Fix "Text file busy" exception in atomic_move (#9526) (#17204) tempfile.NamedTemporaryFile keeps a file handle causing os.rename() to fail with windows based vboxfs: [Errno 26] Text file busy. Changed NamedTemporaryFile to mkstemp() and added a finally block to unlink the temp file in each and every case. --- lib/ansible/module_utils/basic.py | 84 ++++++++++++++------------- test/units/module_utils/test_basic.py | 26 ++++----- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 883e488a3f..6f85f5af1e 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1944,55 +1944,59 @@ class AnsibleModule(object): dest_dir = os.path.dirname(dest) dest_file = os.path.basename(dest) try: - tmp_dest = tempfile.NamedTemporaryFile( + tmp_dest_fd, tmp_dest_name = tempfile.mkstemp( prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file) except (OSError, IOError): e = get_exception() self.fail_json(msg='The destination directory (%s) is not writable by the current user. Error was: %s' % (dest_dir, e)) - try: # leaves tmp file behind when sudo and not root - if switched_user and os.getuid() != 0: - # cleanup will happen by 'rm' of tempdir - # copy2 will preserve some metadata - shutil.copy2(src, tmp_dest.name) - else: - shutil.move(src, tmp_dest.name) - if self.selinux_enabled(): - self.set_context_if_different( - tmp_dest.name, context, False) + try: try: - tmp_stat = os.stat(tmp_dest.name) - if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid): - os.chown(tmp_dest.name, dest_stat.st_uid, dest_stat.st_gid) - except OSError: - e = get_exception() - if e.errno != errno.EPERM: - raise - os.rename(tmp_dest.name, dest) - except (shutil.Error, OSError, IOError): - e = get_exception() - # sadly there are some situations where we cannot ensure atomicity, but only if - # the user insists and we get the appropriate error we update the file unsafely - if unsafe_writes and e.errno == errno.EBUSY: - #TODO: issue warning that this is an unsafe operation, but doing it cause user insists + # close tmp file handle before file operations to prevent text file busy errors on vboxfs synced folders (windows host) + os.close(tmp_dest_fd) + # leaves tmp file behind when sudo and not root + if switched_user and os.getuid() != 0: + # cleanup will happen by 'rm' of tempdir + # copy2 will preserve some metadata + shutil.copy2(src, tmp_dest_name) + else: + shutil.move(src, tmp_dest_name) + if self.selinux_enabled(): + self.set_context_if_different( + tmp_dest_name, context, False) try: - try: - out_dest = open(dest, 'wb') - in_src = open(src, 'rb') - shutil.copyfileobj(in_src, out_dest) - finally: # assuring closed files in 2.4 compatible way - if out_dest: - out_dest.close() - if in_src: - in_src.close() - except (shutil.Error, OSError, IOError): + tmp_stat = os.stat(tmp_dest_name) + if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid): + os.chown(tmp_dest_name, dest_stat.st_uid, dest_stat.st_gid) + except OSError: e = get_exception() - self.fail_json(msg='Could not write data to file (%s) from (%s): %s' % (dest, src, e)) + if e.errno != errno.EPERM: + raise + os.rename(tmp_dest_name, dest) + except (shutil.Error, OSError, IOError): + e = get_exception() + # sadly there are some situations where we cannot ensure atomicity, but only if + # the user insists and we get the appropriate error we update the file unsafely + if unsafe_writes and e.errno == errno.EBUSY: + #TODO: issue warning that this is an unsafe operation, but doing it cause user insists + try: + try: + out_dest = open(dest, 'wb') + in_src = open(src, 'rb') + shutil.copyfileobj(in_src, out_dest) + finally: # assuring closed files in 2.4 compatible way + if out_dest: + out_dest.close() + if in_src: + in_src.close() + except (shutil.Error, OSError, IOError): + e = get_exception() + self.fail_json(msg='Could not write data to file (%s) from (%s): %s' % (dest, src, e)) - else: - self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e)) - - self.cleanup(tmp_dest.name) + else: + self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e)) + finally: + self.cleanup(tmp_dest_name) if creating: # make sure the file has the correct permissions diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index a68352aea6..7f88262261 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -741,7 +741,7 @@ class TestModuleUtilsBasic(ModuleTestCase): with patch('os.lchown', side_effect=OSError) as m: self.assertRaises(SystemExit, am.set_group_if_different, '/path/to/file', 'root', False) - @patch('tempfile.NamedTemporaryFile') + @patch('tempfile.mkstemp') @patch('os.umask') @patch('shutil.copyfileobj') @patch('shutil.move') @@ -755,8 +755,10 @@ class TestModuleUtilsBasic(ModuleTestCase): @patch('os.chmod') @patch('os.stat') @patch('os.path.exists') + @patch('os.close') def test_module_utils_basic_ansible_module_atomic_move( self, + _os_close, _os_path_exists, _os_stat, _os_chmod, @@ -770,7 +772,7 @@ class TestModuleUtilsBasic(ModuleTestCase): _shutil_move, _shutil_copyfileobj, _os_umask, - _tempfile_NamedTemporaryFile, + _tempfile_mkstemp, ): from ansible.module_utils import basic @@ -903,20 +905,21 @@ class TestModuleUtilsBasic(ModuleTestCase): self.assertRaises(SystemExit, am.atomic_move, '/path/to/src', '/path/to/dest') # next we test with EPERM so it continues to the alternate code for moving - # test with NamedTemporaryFile raising an error first + # test with mkstemp raising an error first _os_path_exists.side_effect = [False, False] _os_getlogin.return_value = 'root' _os_getuid.return_value = 0 + _os_close.return_value = None _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '') _os_umask.side_effect = [18, 0] _os_rename.side_effect = [OSError(errno.EPERM, 'failing with EPERM'), None] - _tempfile_NamedTemporaryFile.return_value = None - _tempfile_NamedTemporaryFile.side_effect = OSError() + _tempfile_mkstemp.return_value = None + _tempfile_mkstemp.side_effect = OSError() am.selinux_enabled.return_value = False self.assertRaises(SystemExit, am.atomic_move, '/path/to/src', '/path/to/dest') # then test with it creating a temp file - _os_path_exists.side_effect = [False, False] + _os_path_exists.side_effect = [False, False, False] _os_getlogin.return_value = 'root' _os_getuid.return_value = 0 _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '') @@ -927,23 +930,20 @@ class TestModuleUtilsBasic(ModuleTestCase): mock_stat3 = MagicMock() _os_stat.return_value = [mock_stat1, mock_stat2, mock_stat3] _os_stat.side_effect = None - mock_tempfile = MagicMock() - mock_tempfile.name = '/path/to/tempfile' - _tempfile_NamedTemporaryFile.return_value = mock_tempfile - _tempfile_NamedTemporaryFile.side_effect = None + _tempfile_mkstemp.return_value = (None, '/path/to/tempfile') + _tempfile_mkstemp.side_effect = None am.selinux_enabled.return_value = False # FIXME: we don't assert anything here yet am.atomic_move('/path/to/src', '/path/to/dest') # same as above, but with selinux enabled - _os_path_exists.side_effect = [False, False] + _os_path_exists.side_effect = [False, False, False] _os_getlogin.return_value = 'root' _os_getuid.return_value = 0 _pwd_getpwuid.return_value = ('root', '', 0, 0, '', '', '') _os_umask.side_effect = [18, 0] _os_rename.side_effect = [OSError(errno.EPERM, 'failing with EPERM'), None] - mock_tempfile = MagicMock() - _tempfile_NamedTemporaryFile.return_value = mock_tempfile + _tempfile_mkstemp.return_value = (None, None) mock_context = MagicMock() am.selinux_default_context.return_value = mock_context am.selinux_enabled.return_value = True From b4a035718ebb3fc7df0f932b2423d7533a1ddfca Mon Sep 17 00:00:00 2001 From: Abhijit Menon-Sen Date: Fri, 26 Aug 2016 20:25:44 +0530 Subject: [PATCH 10/30] Make _display_plugin_load much less noisy There was general consensus that displaying every plugin load on -vvv was *way* too noisy. This commit reformats the log message to be less verbose, and drops it down to debugging-only level. --- lib/ansible/plugins/__init__.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index 5472486730..85e5c2c2a3 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -357,16 +357,15 @@ class PluginLoader: return obj def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): - searched_msg = 'Searching for plugin type %s named \'%s\' in paths: %s' % (class_name, name, self.format_paths(searched_paths)) - loading_msg = 'Loading plugin type %s named \'%s\' from %s' % (class_name, name, path) + msg = 'Loading %s \'%s\' from %s' % (class_name, os.path.basename(name), path) + + if len(searched_paths) > 1: + msg = '%s (searched paths: %s)' % (msg, self.format_paths(searched_paths)) if found_in_cache or class_only: - extra_msg = 'found_in_cache=%s, class_only=%s' % (found_in_cache, class_only) - display.debug('%s %s' % (searched_msg, extra_msg)) - display.debug('%s %s' % (loading_msg, extra_msg)) - else: - display.vvvv(searched_msg) - display.vvv(loading_msg) + msg = '%s (found_in_cache=%s, class_only=%s)' % (msg, found_in_cache, class_only) + + display.debug(msg) def all(self, *args, **kwargs): ''' instantiates all plugins with the same arguments ''' From b4f338bca738cbd3e3b64be2fc9579bcefa28d6e Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Fri, 26 Aug 2016 17:27:41 +0200 Subject: [PATCH 11/30] Add OpenBSD virtualization facts. (#17227) * Add OpenBSD virtualization facts. Patch written by @jasperla. Tested by various people on: - virtualbox - vmware esx(i) + fusion - kvm (smartos + plain linux + a random cloud provider) This patch is already present in the OpenBSD port of ansible. * Rework diff to get rid of extra returns. Requested by @bcoca. While here, use four-space indentations of all code blocks. * Set facts even if no match is found. Discussed with @bcoca. * Find sysctl via get_bin_path(). Requested by @bcoca. * Fail if we do not find a sysctl binary. * Do not fail if a sysctl binary is not found. Just set empty fact values instead. Requested by @bcoca. --- lib/ansible/module_utils/facts.py | 45 +++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index 78c460756d..652aa12a5d 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -3171,8 +3171,49 @@ class OpenBSDVirtual(Virtual): return self.facts def get_virtual_facts(self): - self.facts['virtualization_type'] = '' - self.facts['virtualization_role'] = '' + sysctl_path = self.module.get_bin_path('sysctl') + + if sysctl_path: + rc, out, err = self.module.run_command("%s -n hw.product" % sysctl_path) + if rc != 0: + self.facts['virtualization_type'] = '' + self.facts['virtualization_role'] = '' + elif re.match('(KVM|Bochs|SmartDC).*', out): + self.facts['virtualization_type'] = 'kvm' + self.facts['virtualization_role'] = 'guest' + elif re.match('.*VMware.*', out): + self.facts['virtualization_type'] = 'VMware' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'VirtualBox': + self.facts['virtualization_type'] = 'virtualbox' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'HVM domU': + self.facts['virtualization_type'] = 'xen' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'Parallels': + self.facts['virtualization_type'] = 'parallels' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'RHEV Hypervisor': + self.facts['virtualization_type'] = 'RHEV' + self.facts['virtualization_role'] = 'guest' + else: + # Try harder and see if hw.vendor has anything we could use. + rc, out, err = self.module.run_command("%s -n hw.vendor" % sysctl_path) + if rc != 0: + self.facts['virtualization_type'] = '' + self.facts['virtualization_role'] = '' + elif out.rstrip() == 'QEMU': + self.facts['virtualization_type'] = 'kvm' + self.facts['virtualization_role'] = 'guest' + else: + # Set empty values if we find no match at all. + self.facts['virtualization_type'] = '' + self.facts['virtualization_role'] = '' + else: + # Set empty values if we find no sysctl binary. + self.facts['virtualization_type'] = '' + self.facts['virtualization_role'] = '' + class HPUXVirtual(Virtual): """ From 1139d61d59bdc4ca2113fb6f7f8a9b211f358e00 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 26 Aug 2016 11:39:12 -0400 Subject: [PATCH 12/30] simplified logic paths --- lib/ansible/module_utils/facts.py | 63 +++++++++++++------------------ 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index 652aa12a5d..a173e36dc1 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -3173,47 +3173,38 @@ class OpenBSDVirtual(Virtual): def get_virtual_facts(self): sysctl_path = self.module.get_bin_path('sysctl') + # Set empty values as default + self.facts['virtualization_type'] = '' + self.facts['virtualization_role'] = '' + if sysctl_path: rc, out, err = self.module.run_command("%s -n hw.product" % sysctl_path) - if rc != 0: - self.facts['virtualization_type'] = '' - self.facts['virtualization_role'] = '' - elif re.match('(KVM|Bochs|SmartDC).*', out): - self.facts['virtualization_type'] = 'kvm' - self.facts['virtualization_role'] = 'guest' - elif re.match('.*VMware.*', out): - self.facts['virtualization_type'] = 'VMware' - self.facts['virtualization_role'] = 'guest' - elif out.rstrip() == 'VirtualBox': - self.facts['virtualization_type'] = 'virtualbox' - self.facts['virtualization_role'] = 'guest' - elif out.rstrip() == 'HVM domU': - self.facts['virtualization_type'] = 'xen' - self.facts['virtualization_role'] = 'guest' - elif out.rstrip() == 'Parallels': - self.facts['virtualization_type'] = 'parallels' - self.facts['virtualization_role'] = 'guest' - elif out.rstrip() == 'RHEV Hypervisor': - self.facts['virtualization_type'] = 'RHEV' - self.facts['virtualization_role'] = 'guest' - else: - # Try harder and see if hw.vendor has anything we could use. - rc, out, err = self.module.run_command("%s -n hw.vendor" % sysctl_path) - if rc != 0: - self.facts['virtualization_type'] = '' - self.facts['virtualization_role'] = '' - elif out.rstrip() == 'QEMU': + if rc == 0: + if re.match('(KVM|Bochs|SmartDC).*', out): self.facts['virtualization_type'] = 'kvm' self.facts['virtualization_role'] = 'guest' + elif re.match('.*VMware.*', out): + self.facts['virtualization_type'] = 'VMware' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'VirtualBox': + self.facts['virtualization_type'] = 'virtualbox' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'HVM domU': + self.facts['virtualization_type'] = 'xen' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'Parallels': + self.facts['virtualization_type'] = 'parallels' + self.facts['virtualization_role'] = 'guest' + elif out.rstrip() == 'RHEV Hypervisor': + self.facts['virtualization_type'] = 'RHEV' + self.facts['virtualization_role'] = 'guest' else: - # Set empty values if we find no match at all. - self.facts['virtualization_type'] = '' - self.facts['virtualization_role'] = '' - else: - # Set empty values if we find no sysctl binary. - self.facts['virtualization_type'] = '' - self.facts['virtualization_role'] = '' - + # Try harder and see if hw.vendor has anything we could use. + rc, out, err = self.module.run_command("%s -n hw.vendor" % sysctl_path) + if rc == 0: + if out.rstrip() == 'QEMU': + self.facts['virtualization_type'] = 'kvm' + self.facts['virtualization_role'] = 'guest' class HPUXVirtual(Virtual): """ From a30f545a62a1a4f66afa77b0ea4eef761afc06ba Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Fri, 26 Aug 2016 18:48:27 +0200 Subject: [PATCH 13/30] Do not convert Nonetype to "None" (#17261) If someone use a task with a empty name like this: - name: command: true This will result in displaying 'None' as a task name instead of 'command'. --- lib/ansible/plugins/strategy/free.py | 4 ++-- lib/ansible/plugins/strategy/linear.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ansible/plugins/strategy/free.py b/lib/ansible/plugins/strategy/free.py index 772db5cf5a..7435dba04b 100644 --- a/lib/ansible/plugins/strategy/free.py +++ b/lib/ansible/plugins/strategy/free.py @@ -26,7 +26,7 @@ from ansible.playbook.included_file import IncludedFile from ansible.plugins import action_loader from ansible.plugins.strategy import StrategyBase from ansible.template import Templar -from ansible.compat.six import text_type +from ansible.utils.unicode import to_unicode try: from __main__ import display @@ -109,7 +109,7 @@ class StrategyModule(StrategyBase): display.debug("done getting variables") try: - task.name = text_type(templar.template(task.name, fail_on_undefined=False)) + task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py index fe5dbef67c..3f273606e2 100644 --- a/lib/ansible/plugins/strategy/linear.py +++ b/lib/ansible/plugins/strategy/linear.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.compat.six import iteritems, text_type +from ansible.compat.six import iteritems from ansible.errors import AnsibleError from ansible.executor.play_iterator import PlayIterator @@ -238,7 +238,7 @@ class StrategyModule(StrategyBase): saved_name = task.name display.debug("done copying, going to template now") try: - task.name = text_type(templar.template(task.name, fail_on_undefined=False)) + task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, From eac7caefd80bcddfe22a2fe3ddfcf50453d11466 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Fri, 26 Aug 2016 13:16:18 -0400 Subject: [PATCH 14/30] add commit keyword arg to load_config method this adds a new keyword arg to the load_config method that will control whether or not a loaded configuration is committed on the device --- lib/ansible/module_utils/iosxr.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/ansible/module_utils/iosxr.py b/lib/ansible/module_utils/iosxr.py index c3f50370df..4b54446fff 100644 --- a/lib/ansible/module_utils/iosxr.py +++ b/lib/ansible/module_utils/iosxr.py @@ -64,7 +64,7 @@ class Cli(CliBase): responses = self.execute(cmds) return responses - ### immplementation of netcfg.Config ### + ### implementation of netcfg.Config ### def configure(self, commands, **kwargs): cmds = ['configure terminal'] @@ -81,7 +81,7 @@ class Cli(CliBase): cmd += ' %s' % flags return self.execute([cmd])[0] - def load_config(self, config, replace=False, commit=False, **kwargs): + def load_config(self, config, commit=False, replace=False, comment=None): commands = ['configure terminal'] commands.extend(config) @@ -94,19 +94,22 @@ class Cli(CliBase): if commit: if replace: prompt = re.compile(r'\[no\]:\s$') - cmd = Command('commit replace', prompt=prompt, - response='yes') + commit = 'commit replace' + if comment: + commit += ' comment %s' % comment + cmd = Command(commit, prompt=prompt, response='yes') self.execute([cmd, 'end']) else: - self.execute(['commit', 'end']) + commit = 'commit' + if comment: + commit += ' comment %s' % comment + self.execute([commit, 'end']) + else: + self.execute(['abort']) except NetworkError: self.execute(['abort']) diff = None raise return diff[0] - def save_config(self): - raise NotImplementedError - - Cli = register_transport('cli', default=True)(Cli) From bd9094c9250975e7b556311b5e20562c2cc52af7 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 26 Aug 2016 13:42:13 -0400 Subject: [PATCH 15/30] include_role (role revamp implementation) (#17232) * attempt #11 to role_include * fixes from jimi-c * do not override load_data, move all to load * removed debugging * implemented tasks_from parameter, must break cache * fixed issue with cache and tasks_from * make resolution of from_tasks prioritize literal * avoid role dependency dedupe when include_role * fixed role deps and handlers are now loaded * simplified code, enabled k=v parsing used example from jimi-c * load role defaults for task when include_role * fixed issue with from_Tasks overriding all subdirs * corrected priority order of main candidates * made tasks_from a more generic interface to roles * fix block inheritance and handler order * allow vars: clause into included role * pull vars already processed vs from raw data * fix from jimi-c blocks i broke * added back append for dynamic includes * only allow for basename in from parameter * fix for docs when no default * fixed notes * added include_role to changelog --- CHANGELOG.md | 1 + lib/ansible/cli/doc.py | 6 +- lib/ansible/executor/task_executor.py | 9 +++ lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- lib/ansible/parsing/mod_args.py | 8 +-- lib/ansible/playbook/helpers.py | 21 ++++-- lib/ansible/playbook/role/__init__.py | 47 ++++++++++---- lib/ansible/playbook/role_include.py | 82 ++++++++++++++++++++++++ lib/ansible/plugins/strategy/__init__.py | 5 +- lib/ansible/vars/__init__.py | 10 +-- 11 files changed, 163 insertions(+), 30 deletions(-) create mode 100644 lib/ansible/playbook/role_include.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 83eb3603e6..55f17d9821 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Ansible Changes By Release - ipmi * ipmi_boot * ipmi_power +- include_role - letsencrypt - logicmonitor - logicmonitor_facts diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index 14ce38b809..702b27ff10 100644 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -283,13 +283,15 @@ class DocCLI(CLI): choices = '' if 'choices' in opt: choices = "(Choices: " + ", ".join(str(i) for i in opt['choices']) + ")" + default = '' if 'default' in opt or not required: default = "[Default: " + str(opt.get('default', '(null)')) + "]" text.append(textwrap.fill(CLI.tty_ify(choices + default), limit, initial_indent=opt_indent, subsequent_indent=opt_indent)) if 'notes' in doc and doc['notes'] and len(doc['notes']) > 0: - notes = " ".join(doc['notes']) - text.append("Notes:%s\n" % textwrap.fill(CLI.tty_ify(notes), limit-6, initial_indent=" ", subsequent_indent=opt_indent)) + text.append("Notes:") + for note in doc['notes']: + text.append(textwrap.fill(CLI.tty_ify(note), limit-6, initial_indent=" * ", subsequent_indent=opt_indent)) if 'requirements' in doc and doc['requirements'] is not None and len(doc['requirements']) > 0: req = ", ".join(doc['requirements']) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index a0982d9a8b..295116209f 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -404,6 +404,15 @@ class TaskExecutor: include_file = templar.template(include_file) return dict(include=include_file, include_variables=include_variables) + #TODO: not needed? + # if this task is a IncludeRole, we just return now with a success code so the main thread can expand the task list for the given host + elif self._task.action == 'include_role': + include_variables = self._task.args.copy() + role = include_variables.pop('name') + if not role: + return dict(failed=True, msg="No role was specified to include") + return dict(name=role, include_variables=include_variables) + # Now we do final validation on the task, which sets all fields to their final values. self._task.post_validate(templar=templar) if '_variable_params' in self._task.args: diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 4912ec30a7..91a839f1e3 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 4912ec30a71b09577a78f940c6a772b38b76f1a2 +Subproject commit 91a839f1e3de58be8b981d3278aa5dc8ff59c508 diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index f29efb5626..1aeb9f8a8c 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit f29efb56264a9ad95b97765e367ef5b7915ab877 +Subproject commit 1aeb9f8a8c6d54663bbad6db385f568c04182ec6 diff --git a/lib/ansible/parsing/mod_args.py b/lib/ansible/parsing/mod_args.py index 96fb583a71..a4704bd8cc 100644 --- a/lib/ansible/parsing/mod_args.py +++ b/lib/ansible/parsing/mod_args.py @@ -264,7 +264,6 @@ class ModuleArgsParser: if 'action' in self._task_ds: # an old school 'action' statement thing = self._task_ds['action'] - action, args = self._normalize_parameters(thing, additional_args=additional_args) # local_action if 'local_action' in self._task_ds: @@ -273,19 +272,20 @@ class ModuleArgsParser: raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds) thing = self._task_ds.get('local_action', '') delegate_to = 'localhost' - action, args = self._normalize_parameters(thing, additional_args=additional_args) # module: is the more new-style invocation # walk the input dictionary to see we recognize a module name for (item, value) in iteritems(self._task_ds): - if item in module_loader or item == 'meta' or item == 'include': + if item in module_loader or item in ['meta', 'include', 'include_role']: # finding more than one module name is a problem if action is not None: raise AnsibleParserError("conflicting action statements", obj=self._task_ds) action = item thing = value - action, args = self._normalize_parameters(value, action=action, additional_args=additional_args) + #TODO: find out if we should break here? Currently last matching action, break would make it first one + + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) # if we didn't see any module in the task at all, it's not a task really if action is None: diff --git a/lib/ansible/playbook/helpers.py b/lib/ansible/playbook/helpers.py index fc6837dfbc..123e2d3f98 100644 --- a/lib/ansible/playbook/helpers.py +++ b/lib/ansible/playbook/helpers.py @@ -22,8 +22,7 @@ import os from ansible import constants as C from ansible.compat.six import string_types -from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound -from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleSequence +from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound, AnsibleError try: from __main__ import display @@ -81,6 +80,7 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h from ansible.playbook.handler import Handler from ansible.playbook.task import Task from ansible.playbook.task_include import TaskInclude + from ansible.playbook.role_include import IncludeRole from ansible.playbook.handler_task_include import HandlerTaskInclude from ansible.template import Templar @@ -172,7 +172,7 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h if not found: try: include_target = templar.template(t.args['_raw_params']) - except AnsibleUndefinedVariable as e: + except AnsibleUndefinedVariable: raise AnsibleParserError( "Error when evaluating variable in include name: %s.\n\n" \ "When using static includes, ensure that any variables used in their names are defined in vars/vars_files\n" \ @@ -198,7 +198,7 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h # because the recursive nature of helper methods means we may be loading # nested includes, and we want the include order printed correctly display.display("statically included: %s" % include_file, color=C.COLOR_SKIP) - except AnsibleFileNotFound as e: + except AnsibleFileNotFound: if t.static or \ C.DEFAULT_TASK_INCLUDES_STATIC or \ C.DEFAULT_HANDLER_INCLUDES_STATIC and use_handlers: @@ -258,11 +258,24 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h task_list.extend(included_blocks) else: task_list.append(t) + + elif 'include_role' in task_ds: + task_list.extend( + IncludeRole.load( + task_ds, + block=block, + role=role, + task_include=None, + variable_manager=variable_manager, + loader=loader + ) + ) else: if use_handlers: t = Handler.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader) else: t = Task.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader) + task_list.append(t) return task_list diff --git a/lib/ansible/playbook/role/__init__.py b/lib/ansible/playbook/role/__init__.py index bccf860b26..6f91740d7c 100644 --- a/lib/ansible/playbook/role/__init__.py +++ b/lib/ansible/playbook/role/__init__.py @@ -66,7 +66,7 @@ class Role(Base, Become, Conditional, Taggable): _delegate_to = FieldAttribute(isa='string') _delegate_facts = FieldAttribute(isa='bool', default=False) - def __init__(self, play=None): + def __init__(self, play=None, from_files=None): self._role_name = None self._role_path = None self._role_params = dict() @@ -83,6 +83,10 @@ class Role(Base, Become, Conditional, Taggable): self._had_task_run = dict() self._completed = dict() + if from_files is None: + from_files = {} + self._tasks_from = from_files.get('tasks') + super(Role, self).__init__() def __repr__(self): @@ -92,7 +96,10 @@ class Role(Base, Become, Conditional, Taggable): return self._role_name @staticmethod - def load(role_include, play, parent_role=None): + def load(role_include, play, parent_role=None, from_files=None): + + if from_files is None: + from_files = {} try: # The ROLE_CACHE is a dictionary of role names, with each entry # containing another dictionary corresponding to a set of parameters @@ -104,6 +111,10 @@ class Role(Base, Become, Conditional, Taggable): params['when'] = role_include.when if role_include.tags is not None: params['tags'] = role_include.tags + if from_files is not None: + params['from_files'] = from_files + if role_include.vars: + params['vars'] = role_include.vars hashed_params = hash_params(params) if role_include.role in play.ROLE_CACHE: for (entry, role_obj) in iteritems(play.ROLE_CACHE[role_include.role]): @@ -112,7 +123,7 @@ class Role(Base, Become, Conditional, Taggable): role_obj.add_parent(parent_role) return role_obj - r = Role(play=play) + r = Role(play=play, from_files=from_files) r._load_role_data(role_include, parent_role=parent_role) if role_include.role not in play.ROLE_CACHE: @@ -163,7 +174,7 @@ class Role(Base, Become, Conditional, Taggable): else: self._metadata = RoleMetadata() - task_data = self._load_role_yaml('tasks') + task_data = self._load_role_yaml('tasks', main=self._tasks_from) if task_data: try: self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager) @@ -190,23 +201,36 @@ class Role(Base, Become, Conditional, Taggable): elif not isinstance(self._default_vars, dict): raise AnsibleParserError("The defaults/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name) - def _load_role_yaml(self, subdir): + def _load_role_yaml(self, subdir, main=None): file_path = os.path.join(self._role_path, subdir) if self._loader.path_exists(file_path) and self._loader.is_directory(file_path): - main_file = self._resolve_main(file_path) + main_file = self._resolve_main(file_path, main) if self._loader.path_exists(main_file): return self._loader.load_from_file(main_file) return None - def _resolve_main(self, basepath): + def _resolve_main(self, basepath, main=None): ''' flexibly handle variations in main filenames ''' + + post = False + # allow override if set, otherwise use default + if main is None: + main = 'main' + post = True + + bare_main = os.path.join(basepath, main) + possible_mains = ( - os.path.join(basepath, 'main.yml'), - os.path.join(basepath, 'main.yaml'), - os.path.join(basepath, 'main.json'), - os.path.join(basepath, 'main'), + os.path.join(basepath, '%s.yml' % main), + os.path.join(basepath, '%s.yaml' % main), + os.path.join(basepath, '%s.json' % main), ) + if post: + possible_mains = possible_mains + (bare_main,) + else: + possible_mains = (bare_main,) + possible_mains + if sum([self._loader.is_file(x) for x in possible_mains]) > 1: raise AnsibleError("found multiple main files at %s, only one allowed" % (basepath)) else: @@ -274,6 +298,7 @@ class Role(Base, Become, Conditional, Taggable): for dep in self.get_all_dependencies(): all_vars = combine_vars(all_vars, dep.get_vars(include_params=include_params)) + all_vars = combine_vars(all_vars, self.vars) all_vars = combine_vars(all_vars, self._role_vars) if include_params: all_vars = combine_vars(all_vars, self.get_role_params(dep_chain=dep_chain)) diff --git a/lib/ansible/playbook/role_include.py b/lib/ansible/playbook/role_include.py new file mode 100644 index 0000000000..ce2d6aec8f --- /dev/null +++ b/lib/ansible/playbook/role_include.py @@ -0,0 +1,82 @@ +# Copyright (c) 2012 Red Hat, Inc. All rights reserved. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from os.path import basename + +from ansible.playbook.attribute import FieldAttribute +from ansible.playbook.task import Task +from ansible.playbook.role import Role +from ansible.playbook.role.include import RoleInclude + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + +__all__ = ['IncludeRole'] + + +class IncludeRole(Task): + + """ + A Role include is derived from a regular role to handle the special + circumstances related to the `- include_role: ...` + """ + + # ================================================================================= + # ATTRIBUTES + + _name = FieldAttribute(isa='string', default=None) + _tasks_from = FieldAttribute(isa='string', default=None) + + # these should not be changeable? + _static = FieldAttribute(isa='bool', default=False) + _private = FieldAttribute(isa='bool', default=True) + + @staticmethod + def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None): + + r = IncludeRole().load_data(data, variable_manager=variable_manager, loader=loader) + args = r.preprocess_data(data).get('args', dict()) + + ri = RoleInclude.load(args.get('name'), play=block._play, variable_manager=variable_manager, loader=loader) + ri.vars.update(r.vars) + + # build options for roles + from_files = {} + if args.get('tasks_from'): + from_files['tasks'] = basename(args.get('tasks_from')) + + #build role + actual_role = Role.load(ri, block._play, parent_role=role, from_files=from_files) + + # compile role + blocks = actual_role.compile(play=block._play) + + # set parent to ensure proper inheritance + for b in blocks: + b._parent = block + + # updated available handlers in play + block._play.handlers = block._play.handlers + actual_role.get_handler_blocks(play=block._play) + + return blocks diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index 4a7c9f539e..88c5866444 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -40,6 +40,7 @@ from ansible.module_utils.facts import Facts from ansible.playbook.helpers import load_list_of_blocks from ansible.playbook.included_file import IncludedFile from ansible.playbook.task_include import TaskInclude +from ansible.playbook.role_include import IncludeRole from ansible.plugins import action_loader, connection_loader, filter_loader, lookup_loader, module_loader, test_loader from ansible.template import Templar from ansible.utils.unicode import to_unicode @@ -258,7 +259,7 @@ class StrategyBase: def parent_handler_match(target_handler, handler_name): if target_handler: - if isinstance(target_handler, TaskInclude): + if isinstance(target_handler, (TaskInclude, IncludeRole)): try: handler_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, task=target_handler) templar = Templar(loader=self._loader, variables=handler_vars) @@ -477,7 +478,7 @@ class StrategyBase: # If this is a role task, mark the parent role as being run (if # the task was ok or failed, but not skipped or unreachable) - if original_task._role is not None and role_ran: + if original_task._role is not None and role_ran and original_task.action != 'include_role': # lookup the role in the ROLE_CACHE to make sure we're dealing # with the correct object and mark it as executed for (entry, role_obj) in iteritems(iterator._play.ROLE_CACHE[original_task._role._role_name]): diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py index 793e346ada..5f714ce116 100644 --- a/lib/ansible/vars/__init__.py +++ b/lib/ansible/vars/__init__.py @@ -232,11 +232,11 @@ class VariableManager: for role in play.get_roles(): all_vars = combine_vars(all_vars, role.get_default_vars()) - # if we have a task in this context, and that task has a role, make - # sure it sees its defaults above any other roles, as we previously - # (v1) made sure each task had a copy of its roles default vars - if task and task._role is not None: - all_vars = combine_vars(all_vars, task._role.get_default_vars(dep_chain=task.get_dep_chain())) + # if we have a task in this context, and that task has a role, make + # sure it sees its defaults above any other roles, as we previously + # (v1) made sure each task had a copy of its roles default vars + if task and task._role is not None and (play or task.action == 'include_role'): + all_vars = combine_vars(all_vars, task._role.get_default_vars(dep_chain=task.get_dep_chain())) if host: # next, if a host is specified, we load any vars from group_vars From c755ae6a1dfbea3b9b84b1dde11822dec9c0695f Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 26 Aug 2016 14:12:15 -0400 Subject: [PATCH 16/30] updated sub refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 91a839f1e3..6eab2b3d40 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 91a839f1e3de58be8b981d3278aa5dc8ff59c508 +Subproject commit 6eab2b3d405ec6d011061ae35f6ecc25e98ce8bf diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 1aeb9f8a8c..628ee6864d 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 1aeb9f8a8c6d54663bbad6db385f568c04182ec6 +Subproject commit 628ee6864d4bd6cfe161345ef4d3f7e4fd6f4ddb From be55bd6cdff2b31ee7be62c3d5329bef8e38d4a9 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Fri, 26 Aug 2016 15:09:02 -0400 Subject: [PATCH 17/30] disable prompt timestamps upon successful connection to iosxr --- lib/ansible/module_utils/iosxr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/iosxr.py b/lib/ansible/module_utils/iosxr.py index 4b54446fff..c15edc9748 100644 --- a/lib/ansible/module_utils/iosxr.py +++ b/lib/ansible/module_utils/iosxr.py @@ -55,7 +55,8 @@ class Cli(CliBase): def connect(self, params, **kwargs): super(Cli, self).connect(params, kickstart=False, **kwargs) - self.shell.send('terminal length 0') + self.shell.send(['terminal length 0', + 'terminal exec prompt no-timestamp']) ### implementation of netcli.Cli ### From bb630f52ff6006800a5d2066706901004e2b84d9 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Fri, 26 Aug 2016 15:58:06 -0400 Subject: [PATCH 18/30] fixes iosxr configure method to commit the changes The iosxr configure method did not send the commit command to active the changes after pushed to the remote device. This change address that problem --- lib/ansible/module_utils/iosxr.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/ansible/module_utils/iosxr.py b/lib/ansible/module_utils/iosxr.py index c15edc9748..b4910208f2 100644 --- a/lib/ansible/module_utils/iosxr.py +++ b/lib/ansible/module_utils/iosxr.py @@ -69,7 +69,10 @@ class Cli(CliBase): def configure(self, commands, **kwargs): cmds = ['configure terminal'] + if commands[-1] == 'end': + commands.pop() cmds.extend(to_list(commands)) + cmds.extend(['commit', 'end']) responses = self.execute(cmds) return responses[1:] From f25ec5adb3bb00f2afebc8cc6ed87c23af4b8bd0 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 26 Aug 2016 16:15:55 -0400 Subject: [PATCH 19/30] fix action parsing to avoid conflicts agin --- lib/ansible/parsing/mod_args.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/ansible/parsing/mod_args.py b/lib/ansible/parsing/mod_args.py index a4704bd8cc..dbe85b0bad 100644 --- a/lib/ansible/parsing/mod_args.py +++ b/lib/ansible/parsing/mod_args.py @@ -264,6 +264,8 @@ class ModuleArgsParser: if 'action' in self._task_ds: # an old school 'action' statement thing = self._task_ds['action'] + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) + # local_action if 'local_action' in self._task_ds: @@ -272,6 +274,7 @@ class ModuleArgsParser: raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds) thing = self._task_ds.get('local_action', '') delegate_to = 'localhost' + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) # module: is the more new-style invocation @@ -283,9 +286,8 @@ class ModuleArgsParser: raise AnsibleParserError("conflicting action statements", obj=self._task_ds) action = item thing = value - #TODO: find out if we should break here? Currently last matching action, break would make it first one + action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) - action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args) # if we didn't see any module in the task at all, it's not a task really if action is None: From 4e6a7a9e019e9c475c9a428948eb5f8e43d08a06 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 26 Aug 2016 21:54:06 -0400 Subject: [PATCH 20/30] allow include_role to specify vars/defaults files --- lib/ansible/playbook/role/__init__.py | 8 ++++---- lib/ansible/playbook/role_include.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/ansible/playbook/role/__init__.py b/lib/ansible/playbook/role/__init__.py index 6f91740d7c..d2a3cb5037 100644 --- a/lib/ansible/playbook/role/__init__.py +++ b/lib/ansible/playbook/role/__init__.py @@ -85,7 +85,7 @@ class Role(Base, Become, Conditional, Taggable): if from_files is None: from_files = {} - self._tasks_from = from_files.get('tasks') + self._from_files = from_files super(Role, self).__init__() @@ -174,7 +174,7 @@ class Role(Base, Become, Conditional, Taggable): else: self._metadata = RoleMetadata() - task_data = self._load_role_yaml('tasks', main=self._tasks_from) + task_data = self._load_role_yaml('tasks', main=self._from_files.get('tasks')) if task_data: try: self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager) @@ -189,13 +189,13 @@ class Role(Base, Become, Conditional, Taggable): raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=handler_data) # vars and default vars are regular dictionaries - self._role_vars = self._load_role_yaml('vars') + self._role_vars = self._load_role_yaml('vars', main=self._from_files.get('vars')) if self._role_vars is None: self._role_vars = dict() elif not isinstance(self._role_vars, dict): raise AnsibleParserError("The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name) - self._default_vars = self._load_role_yaml('defaults') + self._default_vars = self._load_role_yaml('defaults', main=self._from_files.get('defaults')) if self._default_vars is None: self._default_vars = dict() elif not isinstance(self._default_vars, dict): diff --git a/lib/ansible/playbook/role_include.py b/lib/ansible/playbook/role_include.py index ce2d6aec8f..7dc928bf7b 100644 --- a/lib/ansible/playbook/role_include.py +++ b/lib/ansible/playbook/role_include.py @@ -63,8 +63,10 @@ class IncludeRole(Task): # build options for roles from_files = {} - if args.get('tasks_from'): - from_files['tasks'] = basename(args.get('tasks_from')) + for key in ['tasks', 'vars', 'defaults']: + from_key = key + '_from' + if args.get(from_key): + from_files[key] = basename(args.get(from_key)) #build role actual_role = Role.load(ri, block._play, parent_role=role, from_files=from_files) From 15f10ab4bc52d5f619fab465f63e5315d6d619db Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Fri, 26 Aug 2016 22:52:12 -0400 Subject: [PATCH 21/30] Simple documentation typo fix, 'iIf' to 'If' --- lib/ansible/module_utils/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 6f85f5af1e..dcc76f633f 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -2028,7 +2028,7 @@ class AnsibleModule(object): :kw path_prefix: If given, additional path to find the command in. This adds to the PATH environment vairable so helper commands in the same directory can also be found - :kw cwd: iIf given, working directory to run the command inside + :kw cwd: If given, working directory to run the command inside :kw use_unsafe_shell: See `args` parameter. Default False :kw prompt_regex: Regex string (not a compiled regex) which can be used to detect prompts in the stdout which would otherwise cause From 457cf498681de02fdf19c71ebbf3ee78814471ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20R=C3=BCetschi?= Date: Sat, 27 Aug 2016 09:42:53 +0200 Subject: [PATCH 22/30] univention: add common code for univention corporate server modules (#16172) * univention: add common code for univention corporate server modules * univention: try import only univention specific libraries * Code Review with @2-B, slight API changes and refactoring. * Added module documentation overview, describing the provided functions * Moved module-global objects into getter functions, so that we don't need to import possibly-unavailable univention modules at the module level. * Renamed some exports for improved consistency: - module_name() -> module_by_name() - orig_ldap -> ldap_module() - ldap -> uldap() Note that this introduces slight API changes from the outside. Instead of directly accessing module properties, you now have module functions with the same name. Examples: - ansible.module_utils.univention.position_base_dn() - ansible.module_utils.univention.config_registry() - ansible.module_utils.univention.base_dn() - ansible.module_utils.univention.config() * module_utils univention: fix library * move module_utils from univention to univention_umc, because python import univention fails if library is called univention * univention_umc: fix intention * univention: change common code to BSD-2-clause --- lib/ansible/module_utils/univention_umc.py | 292 +++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 lib/ansible/module_utils/univention_umc.py diff --git a/lib/ansible/module_utils/univention_umc.py b/lib/ansible/module_utils/univention_umc.py new file mode 100644 index 0000000000..e110c0746e --- /dev/null +++ b/lib/ansible/module_utils/univention_umc.py @@ -0,0 +1,292 @@ +# -*- coding: UTF-8 -*- + +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c) 2016, Adfinis SyGroup AG +# Tobias Rueetschi +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + + +"""Univention Corporate Server (UCS) access module. + +Provides the following functions for working with an UCS server. + + - ldap_search(filter, base=None, attr=None) + Search the LDAP via Univention's LDAP wrapper (ULDAP) + + - config_registry() + Return the UCR registry object + + - base_dn() + Return the configured Base DN according to the UCR + + - uldap() + Return a handle to the ULDAP LDAP wrapper + + - umc_module_for_add(module, container_dn, superordinate=None) + Return a UMC module for creating a new object of the given type + + - umc_module_for_edit(module, object_dn, superordinate=None) + Return a UMC module for editing an existing object of the given type + + +Any other module is not part of the "official" API and may change at any time. +""" + +import re + + +__all__ = [ + 'ldap_search', + 'config_registry', + 'base_dn', + 'uldap', + 'umc_module_for_add', + 'umc_module_for_edit', +] + + +_singletons = {} + + +def ldap_module(): + import ldap as orig_ldap + return orig_ldap + + +def _singleton(name, constructor): + if name in _singletons: + return _singletons[name] + _singletons[name] = constructor() + return _singletons[name] + + +def config_registry(): + + def construct(): + import univention.config_registry + ucr = univention.config_registry.ConfigRegistry() + ucr.load() + return ucr + + return _singleton('config_registry', construct) + + +def base_dn(): + return config_registry()['ldap/base'] + + +def uldap(): + "Return a configured univention uldap object" + + def construct(): + try: + secret_file = open('/etc/ldap.secret', 'r') + bind_dn = 'cn=admin,{}'.format(base_dn()) + except IOError: # pragma: no cover + secret_file = open('/etc/machine.secret', 'r') + bind_dn = config_registry()["ldap/hostdn"] + pwd_line = secret_file.readline() + pwd = re.sub('\n', '', pwd_line) + + import univention.admin.uldap + return univention.admin.uldap.access( + host = config_registry()['ldap/master'], + base = base_dn(), + binddn = bind_dn, + bindpw = pwd, + start_tls = 1 + ) + + return _singleton('uldap', construct) + + +def config(): + def construct(): + import univention.admin.config + return univention.admin.config.config() + return _singleton('config', construct) + + +def init_modules(): + def construct(): + import univention.admin.modules + univention.admin.modules.update() + return True + return _singleton('modules_initialized', construct) + + +def position_base_dn(): + def construct(): + import univention.admin.uldap + return univention.admin.uldap.position(base_dn()) + return _singleton('position_base_dn', construct) + + +def ldap_dn_tree_parent(dn, count=1): + dn_array = dn.split(',') + dn_array[0:count] = [] + return ','.join(dn_array) + + +def ldap_search(filter, base=None, attr=None): + """Replaces uldaps search and uses a generator. + !! Arguments are not the same.""" + + if base is None: + base = base_dn() + msgid = uldap().lo.lo.search( + base, + ldap_module().SCOPE_SUBTREE, + filterstr=filter, + attrlist=attr + ) + # I used to have a try: finally: here but there seems to be a bug in python + # which swallows the KeyboardInterrupt + # The abandon now doesn't make too much sense + while True: + result_type, result_data = uldap().lo.lo.result(msgid, all=0) + if not result_data: + break + if result_type is ldap_module().RES_SEARCH_RESULT: # pragma: no cover + break + else: + if result_type is ldap_module().RES_SEARCH_ENTRY: + for res in result_data: + yield res + uldap().lo.lo.abandon(msgid) + + +def module_by_name(module_name_): + """Returns an initialized UMC module, identified by the given name. + + The module is a module specification according to the udm commandline. + Example values are: + * users/user + * shares/share + * groups/group + + If the module does not exist, a KeyError is raised. + + The modules are cached, so they won't be re-initialized + in subsequent calls. + """ + + def construct(): + import univention.admin.modules + init_modules() + module = univention.admin.modules.get(module_name_) + univention.admin.modules.init(uldap(), position_base_dn(), module) + return module + + return _singleton('module/%s' % module_name_, construct) + + +def get_umc_admin_objects(): + """Convenience accessor for getting univention.admin.objects. + + This implements delayed importing, so the univention.* modules + are not loaded until this function is called. + """ + import univention.admin + return univention.admin.objects + + +def umc_module_for_add(module, container_dn, superordinate=None): + """Returns an UMC module object prepared for creating a new entry. + + The module is a module specification according to the udm commandline. + Example values are: + * users/user + * shares/share + * groups/group + + The container_dn MUST be the dn of the container (not of the object to + be created itself!). + """ + mod = module_by_name(module) + + position = position_base_dn() + position.setDn(container_dn) + + # config, ldap objects from common module + obj = mod.object(config(), uldap(), position, superordinate=superordinate) + obj.open() + + return obj + + +def umc_module_for_edit(module, object_dn, superordinate=None): + """Returns an UMC module object prepared for editing an existing entry. + + The module is a module specification according to the udm commandline. + Example values are: + * users/user + * shares/share + * groups/group + + The object_dn MUST be the dn of the object itself, not the container! + """ + mod = module_by_name(module) + + objects = get_umc_admin_objects() + + position = position_base_dn() + position.setDn(ldap_dn_tree_parent(object_dn)) + + obj = objects.get( + mod, + config(), + uldap(), + position=position, + superordinate=superordinate, + dn=object_dn + ) + obj.open() + + return obj + + +def create_containers_and_parents(container_dn): + """Create a container and if needed the parents containers""" + import univention.admin.uexceptions as uexcp + assert container_dn.startswith("cn=") + try: + parent = ldap_dn_tree_parent(container_dn) + obj = umc_module_for_add( + 'container/cn', + parent + ) + obj['name'] = container_dn.split(',')[0].split('=')[1] + obj['description'] = "container created by import" + except uexcp.ldapError: + create_containers_and_parents(parent) + obj = umc_module_for_add( + 'container/cn', + parent + ) + obj['name'] = container_dn.split(',')[0].split('=')[1] + obj['description'] = "container created by import" From 5ce032bf6d5364a71fb10b1308ef96251c54a55e Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Sun, 28 Aug 2016 22:58:52 -0400 Subject: [PATCH 23/30] fixes two bugs in the eos shared module * fixes issue with correctly returning the running-config over eapi when a call was made to get_config() * fixes issue the MRO in Cli transport --- lib/ansible/module_utils/eos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/module_utils/eos.py b/lib/ansible/module_utils/eos.py index 0dd58c082d..1a79b948a8 100644 --- a/lib/ansible/module_utils/eos.py +++ b/lib/ansible/module_utils/eos.py @@ -231,12 +231,12 @@ class Eapi(EosConfigMixin): return response['result'] def get_config(self, **kwargs): - return self.run_commands(['show running-config'], format='text')[0] + return self.execute(['show running-config'], format='text')[0]['output'] Eapi = register_transport('eapi')(Eapi) -class Cli(CliBase, EosConfigMixin): +class Cli(EosConfigMixin, CliBase): CLI_PROMPTS_RE = [ re.compile(r"[\r\n]?[\w+\-\.:\/\[\]]+(?:\([^\)]+\)){,3}(?:>|#) ?$"), From 2cb2ba1fe6bfd0a962b5e2807953e71c6a6486ee Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Sun, 28 Aug 2016 23:02:36 -0400 Subject: [PATCH 24/30] changes nxos method for passing kwargs to get_config() This change makes both the Cli and Nxapi objects handle the get_config() method consistently the same --- lib/ansible/module_utils/nxos.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/ansible/module_utils/nxos.py b/lib/ansible/module_utils/nxos.py index 0378b14954..dde97cad3a 100644 --- a/lib/ansible/module_utils/nxos.py +++ b/lib/ansible/module_utils/nxos.py @@ -173,17 +173,15 @@ class Nxapi(object): return responses - ### end of netcli.Cli ### - ### implemention of netcfg.Config ### def configure(self, commands): commands = to_list(commands) return self.execute(commands, output='config') - def get_config(self, **kwargs): + def get_config(self, include_defaults=False): cmd = 'show running-config' - if kwargs.get('include_defaults'): + if include_defaults: cmd += ' all' return self.execute([cmd], output='text')[0] @@ -263,9 +261,9 @@ class Cli(CliBase): responses.pop(0) return responses - def get_config(self, include_defaults=False, **kwargs): + def get_config(self, include_defaults=False): cmd = 'show running-config' - if kwargs.get('include_defaults'): + if include_defaults: cmd += ' all' return self.execute([cmd])[0] From 820260b22d57aca0ed75fdb81c3b65123788e06d Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Sun, 28 Aug 2016 23:03:46 -0400 Subject: [PATCH 25/30] catches timeout error when connecting to remote host in shell This will now catch a timeout error when shell attempts to open the connection to the remove device and gracefully raise it as a ShellError --- lib/ansible/module_utils/shell.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/ansible/module_utils/shell.py b/lib/ansible/module_utils/shell.py index ac7b038686..1094601f40 100644 --- a/lib/ansible/module_utils/shell.py +++ b/lib/ansible/module_utils/shell.py @@ -106,6 +106,11 @@ class Shell(object): raise ShellError("unable to resolve host name") except AuthenticationException: raise ShellError('Unable to authenticate to remote device') + except socket.error: + exc = get_exception() + if exc.errno == 60: + raise ShellError('timeout trying to connect to host') + raise if self.kickstart: self.shell.sendall("\n") From fa804125b5950a1950afba203691d7b3d2155863 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 29 Aug 2016 09:11:40 -0700 Subject: [PATCH 26/30] Python3 fixes and porting (#17271) * Fix to_native call in selinux_context and selinux_default_context to use the error handler correctly. * Port set_mode_if_different to work on python3 * Port atomic_move to work on python3 * Fix check_password_prompt variable which wasn't renamed properly --- lib/ansible/module_utils/basic.py | 77 +++++++++++-------- lib/ansible/plugins/connection/__init__.py | 2 +- .../basic/test_set_mode_if_different.py | 2 +- test/units/module_utils/test_basic.py | 12 +-- 4 files changed, 51 insertions(+), 42 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index dcc76f633f..a0f1f05111 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -808,7 +808,7 @@ class AnsibleModule(object): if not HAVE_SELINUX or not self.selinux_enabled(): return context try: - ret = selinux.matchpathcon(to_native(path, 'strict'), mode) + ret = selinux.matchpathcon(to_native(path, errors='strict'), mode) except OSError: return context if ret[0] == -1: @@ -823,7 +823,7 @@ class AnsibleModule(object): if not HAVE_SELINUX or not self.selinux_enabled(): return context try: - ret = selinux.lgetfilecon_raw(to_native(path, 'strict')) + ret = selinux.lgetfilecon_raw(to_native(path, errors='strict')) except OSError: e = get_exception() if e.errno == errno.ENOENT: @@ -984,8 +984,9 @@ class AnsibleModule(object): return changed def set_mode_if_different(self, path, mode, changed, diff=None): - path = os.path.expanduser(path) - path_stat = os.lstat(path) + b_path = to_bytes(path) + b_path = os.path.expanduser(b_path) + path_stat = os.lstat(b_path) if mode is None: return changed @@ -1024,22 +1025,22 @@ class AnsibleModule(object): # every time try: if hasattr(os, 'lchmod'): - os.lchmod(path, mode) + os.lchmod(b_path, mode) else: - if not os.path.islink(path): - os.chmod(path, mode) + if not os.path.islink(b_path): + os.chmod(b_path, mode) else: # Attempt to set the perms of the symlink but be # careful not to change the perms of the underlying # file while trying - underlying_stat = os.stat(path) - os.chmod(path, mode) - new_underlying_stat = os.stat(path) + underlying_stat = os.stat(b_path) + os.chmod(b_path, mode) + new_underlying_stat = os.stat(b_path) if underlying_stat.st_mode != new_underlying_stat.st_mode: - os.chmod(path, stat.S_IMODE(underlying_stat.st_mode)) + os.chmod(b_path, stat.S_IMODE(underlying_stat.st_mode)) except OSError: e = get_exception() - if os.path.islink(path) and e.errno == errno.EPERM: # Can't set mode on symbolic links + if os.path.islink(b_path) and e.errno == errno.EPERM: # Can't set mode on symbolic links pass elif e.errno in (errno.ENOENT, errno.ELOOP): # Can't set mode on broken symbolic links pass @@ -1049,7 +1050,7 @@ class AnsibleModule(object): e = get_exception() self.fail_json(path=path, msg='chmod failed', details=str(e)) - path_stat = os.lstat(path) + path_stat = os.lstat(b_path) new_mode = stat.S_IMODE(path_stat.st_mode) if new_mode != prev_mode: @@ -1902,11 +1903,13 @@ class AnsibleModule(object): to work around limitations, corner cases and ensure selinux context is saved if possible''' context = None dest_stat = None - if os.path.exists(dest): + b_src = to_bytes(src) + b_dest = to_bytes(dest) + if os.path.exists(b_dest): try: - dest_stat = os.stat(dest) - os.chmod(src, dest_stat.st_mode & PERM_BITS) - os.chown(src, dest_stat.st_uid, dest_stat.st_gid) + dest_stat = os.stat(b_dest) + os.chmod(b_src, dest_stat.st_mode & PERM_BITS) + os.chown(b_src, dest_stat.st_uid, dest_stat.st_gid) except OSError: e = get_exception() if e.errno != errno.EPERM: @@ -1917,7 +1920,7 @@ class AnsibleModule(object): if self.selinux_enabled(): context = self.selinux_default_context(dest) - creating = not os.path.exists(dest) + creating = not os.path.exists(b_dest) try: login_name = os.getlogin() @@ -1933,7 +1936,7 @@ class AnsibleModule(object): try: # Optimistically try a rename, solves some corner cases and can avoid useless work, throws exception if not atomic. - os.rename(src, dest) + os.rename(b_src, b_dest) except (IOError, OSError): e = get_exception() if e.errno not in [errno.EPERM, errno.EXDEV, errno.EACCES, errno.ETXTBSY]: @@ -1941,14 +1944,20 @@ class AnsibleModule(object): # and 26 (text file busy) which happens on vagrant synced folders and other 'exotic' non posix file systems self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e)) else: - dest_dir = os.path.dirname(dest) - dest_file = os.path.basename(dest) + b_dest_dir = os.path.dirname(b_dest) + # Converting from bytes so that if py3, it will be + # surrogateescaped. If py2, it wil be a noop. Converting + # from text strings could mangle filenames on py2) + native_dest_dir = to_native(b_dest_dir) + native_suffix = to_native(os.path.basename(b_dest)) + native_prefix = '.ansible_tmp' try: tmp_dest_fd, tmp_dest_name = tempfile.mkstemp( - prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file) + prefix=native_prefix, dir=native_dest_dir, suffix=native_suffix) except (OSError, IOError): e = get_exception() - self.fail_json(msg='The destination directory (%s) is not writable by the current user. Error was: %s' % (dest_dir, e)) + self.fail_json(msg='The destination directory (%s) is not writable by the current user. Error was: %s' % (os.path.dirname(dest), e)) + b_tmp_dest_name = to_bytes(tmp_dest_name) try: try: @@ -1958,21 +1967,21 @@ class AnsibleModule(object): if switched_user and os.getuid() != 0: # cleanup will happen by 'rm' of tempdir # copy2 will preserve some metadata - shutil.copy2(src, tmp_dest_name) + shutil.copy2(b_src, b_tmp_dest_name) else: - shutil.move(src, tmp_dest_name) + shutil.move(b_src, b_tmp_dest_name) if self.selinux_enabled(): self.set_context_if_different( - tmp_dest_name, context, False) + b_tmp_dest_name, context, False) try: - tmp_stat = os.stat(tmp_dest_name) + tmp_stat = os.stat(b_tmp_dest_name) if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid): - os.chown(tmp_dest_name, dest_stat.st_uid, dest_stat.st_gid) + os.chown(b_tmp_dest_name, dest_stat.st_uid, dest_stat.st_gid) except OSError: e = get_exception() if e.errno != errno.EPERM: raise - os.rename(tmp_dest_name, dest) + os.rename(b_tmp_dest_name, b_dest) except (shutil.Error, OSError, IOError): e = get_exception() # sadly there are some situations where we cannot ensure atomicity, but only if @@ -1981,8 +1990,8 @@ class AnsibleModule(object): #TODO: issue warning that this is an unsafe operation, but doing it cause user insists try: try: - out_dest = open(dest, 'wb') - in_src = open(src, 'rb') + out_dest = open(b_dest, 'wb') + in_src = open(b_src, 'rb') shutil.copyfileobj(in_src, out_dest) finally: # assuring closed files in 2.4 compatible way if out_dest: @@ -1996,16 +2005,16 @@ class AnsibleModule(object): else: self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e)) finally: - self.cleanup(tmp_dest_name) + self.cleanup(b_tmp_dest_name) if creating: # make sure the file has the correct permissions # based on the current value of umask umask = os.umask(0) os.umask(umask) - os.chmod(dest, DEFAULT_PERM & ~umask) + os.chmod(b_dest, DEFAULT_PERM & ~umask) if switched_user: - os.chown(dest, os.getuid(), os.getgid()) + os.chown(b_dest, os.getuid(), os.getgid()) if self.selinux_enabled(): # rename might not preserve context diff --git a/lib/ansible/plugins/connection/__init__.py b/lib/ansible/plugins/connection/__init__.py index ec3289db97..edd6b498e8 100644 --- a/lib/ansible/plugins/connection/__init__.py +++ b/lib/ansible/plugins/connection/__init__.py @@ -254,7 +254,7 @@ class ConnectionBase(with_metaclass(ABCMeta, object)): b_prompt = to_bytes(self._play_context.prompt) return b_output.startswith(b_prompt) else: - return self._play_context.prompt(output) + return self._play_context.prompt(b_output) def check_incorrect_password(self, b_output): b_incorrect_password = to_bytes(gettext.dgettext(self._play_context.become_method, C.BECOME_ERROR_STRINGS[self._play_context.become_method])) diff --git a/test/units/module_utils/basic/test_set_mode_if_different.py b/test/units/module_utils/basic/test_set_mode_if_different.py index d36a4a3622..f8caea6d36 100644 --- a/test/units/module_utils/basic/test_set_mode_if_different.py +++ b/test/units/module_utils/basic/test_set_mode_if_different.py @@ -68,7 +68,7 @@ def _check_mode_changed_to_0660(self, mode): with patch('os.lstat', side_effect=[self.mock_stat1, self.mock_stat2, self.mock_stat2]) as m_lstat: with patch('os.lchmod', return_value=None, create=True) as m_lchmod: self.assertEqual(self.am.set_mode_if_different('/path/to/file', mode, False), True) - m_lchmod.assert_called_with('/path/to/file', 0o660) + m_lchmod.assert_called_with(b'/path/to/file', 0o660) def _check_mode_unchanged_when_already_0660(self, mode): # Note: This is for checking that all the different ways of specifying diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index 7f88262261..a7fc328d64 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -804,8 +804,8 @@ class TestModuleUtilsBasic(ModuleTestCase): _os_chown.reset_mock() am.set_context_if_different.reset_mock() am.atomic_move('/path/to/src', '/path/to/dest') - _os_rename.assert_called_with('/path/to/src', '/path/to/dest') - self.assertEqual(_os_chmod.call_args_list, [call('/path/to/dest', basic.DEFAULT_PERM & ~18)]) + _os_rename.assert_called_with(b'/path/to/src', b'/path/to/dest') + self.assertEqual(_os_chmod.call_args_list, [call(b'/path/to/dest', basic.DEFAULT_PERM & ~18)]) # same as above, except selinux_enabled _os_path_exists.side_effect = [False, False] @@ -822,8 +822,8 @@ class TestModuleUtilsBasic(ModuleTestCase): am.set_context_if_different.reset_mock() am.selinux_default_context.reset_mock() am.atomic_move('/path/to/src', '/path/to/dest') - _os_rename.assert_called_with('/path/to/src', '/path/to/dest') - self.assertEqual(_os_chmod.call_args_list, [call('/path/to/dest', basic.DEFAULT_PERM & ~18)]) + _os_rename.assert_called_with(b'/path/to/src', b'/path/to/dest') + self.assertEqual(_os_chmod.call_args_list, [call(b'/path/to/dest', basic.DEFAULT_PERM & ~18)]) self.assertEqual(am.selinux_default_context.call_args_list, [call('/path/to/dest')]) self.assertEqual(am.set_context_if_different.call_args_list, [call('/path/to/dest', mock_context, False)]) @@ -846,7 +846,7 @@ class TestModuleUtilsBasic(ModuleTestCase): _os_chown.reset_mock() am.set_context_if_different.reset_mock() am.atomic_move('/path/to/src', '/path/to/dest') - _os_rename.assert_called_with('/path/to/src', '/path/to/dest') + _os_rename.assert_called_with(b'/path/to/src', b'/path/to/dest') # dest missing, selinux enabled _os_path_exists.side_effect = [True, True] @@ -868,7 +868,7 @@ class TestModuleUtilsBasic(ModuleTestCase): am.set_context_if_different.reset_mock() am.selinux_default_context.reset_mock() am.atomic_move('/path/to/src', '/path/to/dest') - _os_rename.assert_called_with('/path/to/src', '/path/to/dest') + _os_rename.assert_called_with(b'/path/to/src', b'/path/to/dest') self.assertEqual(am.selinux_context.call_args_list, [call('/path/to/dest')]) self.assertEqual(am.set_context_if_different.call_args_list, [call('/path/to/dest', mock_context, False)]) From b8a8027b7debb67dd1ee012879314e9737677ba0 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 29 Aug 2016 09:12:37 -0700 Subject: [PATCH 27/30] We've decided that python-3.5 is the minimum python version (#17270) --- .travis.yml | 2 - Makefile | 2 +- docsite/rst/developing_modules_python3.rst | 55 +++++++++++----------- shippable.yml | 2 - tox.ini | 6 +-- 5 files changed, 29 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9731446994..39268bf2ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,6 @@ matrix: python: 2.6 - env: TARGET=sanity TOXENV=py27 python: 2.7 - - env: TARGET=sanity TOXENV=py34 - python: 3.4 - env: TARGET=sanity TOXENV=py35 python: 3.5 - env: TARGET=sanity TOXENV=py24 diff --git a/Makefile b/Makefile index 3c48d46cdb..e1a30f23dc 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ MOCK_CFG ?= NOSETESTS ?= nosetests -NOSETESTS3 ?= nosetests-3.4 +NOSETESTS3 ?= nosetests-3.5 ######################################################## diff --git a/docsite/rst/developing_modules_python3.rst b/docsite/rst/developing_modules_python3.rst index b4e7baeafb..26fdc9ad7f 100644 --- a/docsite/rst/developing_modules_python3.rst +++ b/docsite/rst/developing_modules_python3.rst @@ -13,42 +13,41 @@ factors that make it harder to port them than most code: Which version of Python-3.x and which version of Python-2.x are our minimums? ============================================================================= -The short answer is Python-3.4 and Python-2.4 but please read on for more +The short answer is Python-3.5 and Python-2.4 but please read on for more information. -For Python-3 we are currently using Python-3.4 as a minimum. However, no long -term supported Linux distributions currently ship with Python-3. When that -occurs, we will probably take that as our minimum Python-3 version rather than -Python-3.4. Thus far, Python-3 has been adding small changes that make it -more compatible with Python-2 in its newer versions (For instance, Python-3.5 -added the ability to use percent-formatted byte strings.) so it should be more -pleasant to use a newer version of Python-3 if it's available. At some point -this will change but we'll just have to cross that bridge when we get to it. +For Python-3 we are currently using Python-3.5 as a minimum on both the +controller and the managed nodes. This was chosen as it's the version of +Python3 in Ubuntu-16.04, the first long-term support (LTS) distribution to +ship with Python3 and not Python2. Much of our code would still work with +Python-3.4 but there are always bugfixes and new features in any new upstream +release. Taking advantage of this relatively new version allows us not to +worry about workarounds for problems and missing features in that older +version. -For Python-2 the default is for modules to run on Python-2.4. This allows -users with older distributions that are stuck on Python-2.4 to manage their -machines. Modules are allowed to drop support for Python-2.4 when one of -their dependent libraries require a higher version of python. This is not an -invitation to add unnecessary dependent libraries in order to force your -module to be usable only with a newer version of Python. Instead it is an -acknowledgment that some libraries (for instance, boto3 and docker-py) will -only function with newer Python. +For Python-2, the default is for the controller to run on Python-2.6 and +modules to run on Python-2.4. This allows users with older distributions that +are stuck on Python-2.4 to manage their machines. Modules are allowed to drop +support for Python-2.4 when one of their dependent libraries require a higher +version of python. This is not an invitation to add unnecessary dependent +libraries in order to force your module to be usable only with a newer version +of Python. Instead it is an acknowledgment that some libraries (for instance, +boto3 and docker-py) will only function with newer Python. .. note:: When will we drop support for Python-2.4? The only long term supported distro that we know of with Python-2.4 is - RHEL5 (and its rebuilds like CentOS5) which is supported until April of - 2017. We will likely end our support for Python-2.4 in modules in an - Ansible release around that time. We know of no long term supported - distributions with Python-2.5 so the new minimum Python-2 version will - likely be Python-2.6. This will let us take advantage of the - forwards-compat features of Python-2.6 so porting and maintainance of - Python-2/Python-3 code will be easier after that. + RHEL5 (and its rebuilds like CentOS5) which is supported until April of + 2017. Whatever major release we make in or after April of 2017 (probably + 2.4.0) will no longer have support for Python-2.4 on the managed machines. + Previous major release series's that we support (2.3.x) will continue to + support Python-2.4 on the managed nodes. -.. note:: Ubuntu 16 LTS ships with Python 3.5 + We know of no long term supported distributions with Python-2.5 so the new + minimum Python-2 version will be Python-2.6. This will let us take + advantage of the forwards-compat features of Python-2.6 so porting and + maintainance of Python-2/Python-3 code will be easier after that. - We have ongoing discussions now about taking Python3-3.5 as our minimum - Python3 version. Supporting only Python-2 or only Python-3 ========================================= @@ -152,7 +151,7 @@ dealing with file permissions. (With file permissions a user may be feeding the mode into another program or to another module which doesn't understand the python syntax for octal numbers. ``[zero][digit][digit][digit]`` is understood by most everything and therefore the right way to express octals in -these cisrcumstances. +these circumstances. Bundled six ----------- diff --git a/shippable.yml b/shippable.yml index 49b4fc4191..19feee55ca 100644 --- a/shippable.yml +++ b/shippable.yml @@ -51,8 +51,6 @@ matrix: python: 2.6 - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py27 python: 2.7 - - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py34 - python: 3.4 - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py35 python: 3.5 diff --git a/tox.ini b/tox.ini index 267a5a5a7e..3ac013ad8f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,5 @@ [tox] -envlist = py26,py27,py34,py35 - -[testenv:py34] -deps = -r{toxinidir}/test/utils/tox/requirements-py3.txt +envlist = py26,py27,py35 [testenv:py35] deps = -r{toxinidir}/test/utils/tox/requirements-py3.txt @@ -14,7 +11,6 @@ commands = python --version py26: python -m compileall -fq -x 'test/samples|contrib/inventory/vagrant.py' lib test contrib py27: python -m compileall -fq -x 'test/samples' lib test contrib - py34: python -m compileall -fq -x 'test/samples|lib/ansible/modules' lib test contrib py35: python -m compileall -fq -x 'test/samples|lib/ansible/modules' lib test contrib make tests From 8210260d94eaafe34d288f70a20248e1376a9f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Moser?= Date: Mon, 29 Aug 2016 18:42:47 +0200 Subject: [PATCH 28/30] changelog: update new modules from module-extras (#17286) --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55f17d9821..8ad5f63228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Ansible Changes By Release * Added the `listen` feature for modules. This feature allows tasks to more easily notify multiple handlers, as well as making it easier for handlers from decoupled roles to be notified. * Added support for binary modules -* `raw` now returns `changed: true` to be consistent with shell/command/script modules. Add `changed_when: false` to `raw` tasks to restore the pre-2.2 behavior if necessary. +* `raw` now returns `changed: true` to be consistent with shell/command/script modules. Add `changed_when: false` to `raw` tasks to restore the pre-2.2 behavior if necessary. * The service module has been changed to use system specific modules if they exist and fallback to the old service module if they cannot be found or detected. * Several Windows facts were modified or renamed for consistency with their Unix counterparts, and many new facts were added. If your playbooks rely on any of the following keys, please ensure they are using the correct key names and/or values: - ansible_date_time.date (changed to use yyyy-mm-dd format instead of default system-locale format) @@ -32,6 +32,11 @@ Ansible Changes By Release - cloudstack * cs_router * cs_snapshot_policy +- f5: + * bigip_device_dns + * bigip_device_ntp + * bigip_device_sshd + * bigip_irule - github_key - google * gcdns_record @@ -40,6 +45,7 @@ Ansible Changes By Release * ipmi_boot * ipmi_power - include_role +- jenkins_plugin - letsencrypt - logicmonitor - logicmonitor_facts @@ -60,6 +66,8 @@ Ansible Changes By Release * smartos_image_facts - systemd - telegram +- univention + * udm_user - vmware * vmware_guest * vmware_local_user_manager From 54d3a977f2b337cbc3bc693cd355405bf87dcb2f Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 29 Aug 2016 13:21:02 -0700 Subject: [PATCH 29/30] Update submodule refs --- lib/ansible/modules/core | 2 +- lib/ansible/modules/extras | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 6eab2b3d40..5310bab12f 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 6eab2b3d405ec6d011061ae35f6ecc25e98ce8bf +Subproject commit 5310bab12f6013195ac0e770472d593552271b11 diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index 628ee6864d..2ef4a34eee 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit 628ee6864d4bd6cfe161345ef4d3f7e4fd6f4ddb +Subproject commit 2ef4a34eee091449d2a22312e3e15171f8c6d54c From 1c33b5a9f041e0fdc4ef0973c7143c59c67bfeb9 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 29 Aug 2016 14:01:25 -0700 Subject: [PATCH 30/30] fix remote shippable target failures (#17287) cryptography upgrade caused fatal error when pycrypto was not installed --- test/utils/shippable/remote-requirements.txt | 3 --- test/utils/shippable/remote.sh | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/utils/shippable/remote-requirements.txt b/test/utils/shippable/remote-requirements.txt index 5074d71e48..23a9d3a2a8 100644 --- a/test/utils/shippable/remote-requirements.txt +++ b/test/utils/shippable/remote-requirements.txt @@ -1,11 +1,8 @@ cryptography -jinja2 junit-xml ndg-httpsclient pyasn1 pyopenssl -pyyaml requests -setuptools pywinrm xmltodict diff --git a/test/utils/shippable/remote.sh b/test/utils/shippable/remote.sh index 2bcf8ce786..9901f8e20f 100755 --- a/test/utils/shippable/remote.sh +++ b/test/utils/shippable/remote.sh @@ -86,6 +86,7 @@ if [ ${start_instance} ]; then start --id "${instance_id}" "${test_auth}" "${test_platform}" "${test_version}" ${args} fi +pip install "${source_root}" --upgrade pip install -r "${source_root}/test/utils/shippable/remote-requirements.txt" --upgrade pip list