mirror of
https://github.com/ansible-collections/community.general.git
synced 2026-05-06 05:12:45 +00:00
Use locking for concurrent file access (#52567)
* Use locking for concurrent file access This implements locking to be used for modules that are used for concurrent file access, like lineinfile or known_hosts. * Reinstate lock_timeout This commit includes: - New file locking infrastructure for modules - Enable timeout tests - Madifications to support concurrency with lineinfile * Rebase, update changelog and tests We need to specify ansible_python_interpreter to avoid running interpreter discovery and selecting the incorrect interpreter. Remove the import of lock in known_hosts since it is not used.
This commit is contained in:
Dag Wieers
Sam Doran
@@ -184,6 +184,13 @@ EXAMPLES = r'''
|
||||
line: 192.168.1.99 foo.lab.net foo
|
||||
create: yes
|
||||
|
||||
# Fully quoted because of the ': ' on the line. See the Gotchas in the YAML docs.
|
||||
- lineinfile:
|
||||
path: /etc/sudoers
|
||||
state: present
|
||||
regexp: '^%wheel\s'
|
||||
line: '%wheel ALL=(ALL) NOPASSWD: ALL'
|
||||
|
||||
# NOTE: Yaml requires escaping backslashes in double quotes but not in single quotes
|
||||
- name: Ensure the JBoss memory settings are exactly as needed
|
||||
lineinfile:
|
||||
@@ -208,6 +215,7 @@ import tempfile
|
||||
|
||||
# import module snippets
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
from ansible.module_utils.common.file import open_locked
|
||||
from ansible.module_utils.six import b
|
||||
from ansible.module_utils._text import to_bytes, to_native
|
||||
|
||||
@@ -265,141 +273,148 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create,
|
||||
os.makedirs(b_destpath)
|
||||
except Exception as e:
|
||||
module.fail_json(msg='Error creating %s Error code: %s Error description: %s' % (b_destpath, e[0], e[1]))
|
||||
# destination must exist to be able to lock it
|
||||
if not module.check_mode:
|
||||
open(b_dest, 'ab').close()
|
||||
|
||||
b_lines = []
|
||||
else:
|
||||
with open(b_dest, 'rb') as f:
|
||||
b_lines = f.readlines()
|
||||
b_lines = None
|
||||
|
||||
if module._diff:
|
||||
diff['before'] = to_native(b('').join(b_lines))
|
||||
# NOTE: Avoid opening the same file in this context !
|
||||
with open_locked(dest, module.check_mode) as fd:
|
||||
if b_lines is None:
|
||||
b_lines = fd.readlines()
|
||||
|
||||
if regexp is not None:
|
||||
bre_m = re.compile(to_bytes(regexp, errors='surrogate_or_strict'))
|
||||
if module._diff:
|
||||
diff['before'] = to_native(b('').join(b_lines))
|
||||
|
||||
if insertafter not in (None, 'BOF', 'EOF'):
|
||||
bre_ins = re.compile(to_bytes(insertafter, errors='surrogate_or_strict'))
|
||||
elif insertbefore not in (None, 'BOF'):
|
||||
bre_ins = re.compile(to_bytes(insertbefore, errors='surrogate_or_strict'))
|
||||
else:
|
||||
bre_ins = None
|
||||
|
||||
# index[0] is the line num where regexp has been found
|
||||
# index[1] is the line num where insertafter/inserbefore has been found
|
||||
index = [-1, -1]
|
||||
m = None
|
||||
b_line = to_bytes(line, errors='surrogate_or_strict')
|
||||
for lineno, b_cur_line in enumerate(b_lines):
|
||||
if regexp is not None:
|
||||
match_found = bre_m.search(b_cur_line)
|
||||
bre_m = re.compile(to_bytes(regexp, errors='surrogate_or_strict'))
|
||||
|
||||
if insertafter not in (None, 'BOF', 'EOF'):
|
||||
bre_ins = re.compile(to_bytes(insertafter, errors='surrogate_or_strict'))
|
||||
elif insertbefore not in (None, 'BOF'):
|
||||
bre_ins = re.compile(to_bytes(insertbefore, errors='surrogate_or_strict'))
|
||||
else:
|
||||
match_found = b_line == b_cur_line.rstrip(b('\r\n'))
|
||||
if match_found:
|
||||
index[0] = lineno
|
||||
m = match_found
|
||||
elif bre_ins is not None and bre_ins.search(b_cur_line):
|
||||
if insertafter:
|
||||
# + 1 for the next line
|
||||
index[1] = lineno + 1
|
||||
if firstmatch:
|
||||
break
|
||||
if insertbefore:
|
||||
# index[1] for the previous line
|
||||
index[1] = lineno
|
||||
if firstmatch:
|
||||
break
|
||||
bre_ins = None
|
||||
|
||||
msg = ''
|
||||
changed = False
|
||||
b_linesep = to_bytes(os.linesep, errors='surrogate_or_strict')
|
||||
# Exact line or Regexp matched a line in the file
|
||||
if index[0] != -1:
|
||||
if backrefs:
|
||||
b_new_line = m.expand(b_line)
|
||||
else:
|
||||
# Don't do backref expansion if not asked.
|
||||
b_new_line = b_line
|
||||
# index[0] is the line num where regexp has been found
|
||||
# index[1] is the line num where insertafter/inserbefore has been found
|
||||
index = [-1, -1]
|
||||
m = None
|
||||
b_line = to_bytes(line, errors='surrogate_or_strict')
|
||||
for lineno, b_cur_line in enumerate(b_lines):
|
||||
if regexp is not None:
|
||||
match_found = bre_m.search(b_cur_line)
|
||||
else:
|
||||
match_found = b_line == b_cur_line.rstrip(b('\r\n'))
|
||||
if match_found:
|
||||
index[0] = lineno
|
||||
m = match_found
|
||||
elif bre_ins is not None and bre_ins.search(b_cur_line):
|
||||
if insertafter:
|
||||
# + 1 for the next line
|
||||
index[1] = lineno + 1
|
||||
if firstmatch:
|
||||
break
|
||||
if insertbefore:
|
||||
# index[1] for the previous line
|
||||
index[1] = lineno
|
||||
if firstmatch:
|
||||
break
|
||||
|
||||
if not b_new_line.endswith(b_linesep):
|
||||
b_new_line += b_linesep
|
||||
msg = ''
|
||||
changed = False
|
||||
b_linesep = to_bytes(os.linesep, errors='surrogate_or_strict')
|
||||
# Exact line or Regexp matched a line in the file
|
||||
if index[0] != -1:
|
||||
if backrefs:
|
||||
b_new_line = m.expand(b_line)
|
||||
else:
|
||||
# Don't do backref expansion if not asked.
|
||||
b_new_line = b_line
|
||||
|
||||
# If no regexp was given and no line match is found anywhere in the file,
|
||||
# insert the line appropriately if using insertbefore or insertafter
|
||||
if regexp is None and m is None:
|
||||
if not b_new_line.endswith(b_linesep):
|
||||
b_new_line += b_linesep
|
||||
|
||||
# Insert lines
|
||||
if insertafter and insertafter != 'EOF':
|
||||
# Ensure there is a line separator after the found string
|
||||
# at the end of the file.
|
||||
if b_lines and not b_lines[-1][-1:] in (b('\n'), b('\r')):
|
||||
b_lines[-1] = b_lines[-1] + b_linesep
|
||||
# If no regexp was given and no line match is found anywhere in the file,
|
||||
# insert the line appropriately if using insertbefore or insertafter
|
||||
if regexp is None and m is None:
|
||||
|
||||
# If the line to insert after is at the end of the file
|
||||
# use the appropriate index value.
|
||||
if len(b_lines) == index[1]:
|
||||
if b_lines[index[1] - 1].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.append(b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
elif b_lines[index[1]].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
# Insert lines
|
||||
if insertafter and insertafter != 'EOF':
|
||||
# Ensure there is a line separator after the found string
|
||||
# at the end of the file.
|
||||
if b_lines and not b_lines[-1][-1:] in (b('\n'), b('\r')):
|
||||
b_lines[-1] = b_lines[-1] + b_linesep
|
||||
|
||||
elif insertbefore and insertbefore != 'BOF':
|
||||
# If the line to insert before is at the beginning of the file
|
||||
# use the appropriate index value.
|
||||
if index[1] <= 0:
|
||||
if b_lines[index[1]].rstrip(b('\r\n')) != b_line:
|
||||
# If the line to insert after is at the end of the file
|
||||
# use the appropriate index value.
|
||||
if len(b_lines) == index[1]:
|
||||
if b_lines[index[1] - 1].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.append(b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
elif b_lines[index[1]].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
|
||||
elif b_lines[index[1] - 1].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
elif insertbefore and insertbefore != 'BOF':
|
||||
# If the line to insert before is at the beginning of the file
|
||||
# use the appropriate index value.
|
||||
if index[1] <= 0:
|
||||
if b_lines[index[1]].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
|
||||
elif b_lines[index[0]] != b_new_line:
|
||||
b_lines[index[0]] = b_new_line
|
||||
msg = 'line replaced'
|
||||
elif b_lines[index[1] - 1].rstrip(b('\r\n')) != b_line:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
|
||||
elif b_lines[index[0]] != b_new_line:
|
||||
b_lines[index[0]] = b_new_line
|
||||
msg = 'line replaced'
|
||||
changed = True
|
||||
|
||||
elif backrefs:
|
||||
# Do absolutely nothing, since it's not safe generating the line
|
||||
# without the regexp matching to populate the backrefs.
|
||||
pass
|
||||
# Add it to the beginning of the file
|
||||
elif insertbefore == 'BOF' or insertafter == 'BOF':
|
||||
b_lines.insert(0, b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
# Add it to the end of the file if requested or
|
||||
# if insertafter/insertbefore didn't match anything
|
||||
# (so default behaviour is to add at the end)
|
||||
elif insertafter == 'EOF' or index[1] == -1:
|
||||
|
||||
# If the file is not empty then ensure there's a newline before the added line
|
||||
if b_lines and not b_lines[-1][-1:] in (b('\n'), b('\r')):
|
||||
b_lines.append(b_linesep)
|
||||
|
||||
b_lines.append(b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
# insert matched, but not the regexp
|
||||
else:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
|
||||
elif backrefs:
|
||||
# Do absolutely nothing, since it's not safe generating the line
|
||||
# without the regexp matching to populate the backrefs.
|
||||
pass
|
||||
# Add it to the beginning of the file
|
||||
elif insertbefore == 'BOF' or insertafter == 'BOF':
|
||||
b_lines.insert(0, b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
# Add it to the end of the file if requested or
|
||||
# if insertafter/insertbefore didn't match anything
|
||||
# (so default behaviour is to add at the end)
|
||||
elif insertafter == 'EOF' or index[1] == -1:
|
||||
if module._diff:
|
||||
diff['after'] = to_native(b('').join(b_lines))
|
||||
|
||||
# If the file is not empty then ensure there's a newline before the added line
|
||||
if b_lines and not b_lines[-1][-1:] in (b('\n'), b('\r')):
|
||||
b_lines.append(b_linesep)
|
||||
|
||||
b_lines.append(b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
# insert matched, but not the regexp
|
||||
else:
|
||||
b_lines.insert(index[1], b_line + b_linesep)
|
||||
msg = 'line added'
|
||||
changed = True
|
||||
|
||||
if module._diff:
|
||||
diff['after'] = to_native(b('').join(b_lines))
|
||||
|
||||
backupdest = ""
|
||||
if changed and not module.check_mode:
|
||||
if backup and os.path.exists(b_dest):
|
||||
backupdest = module.backup_local(dest)
|
||||
write_changes(module, b_lines, dest)
|
||||
backupdest = ""
|
||||
if changed and not module.check_mode:
|
||||
if backup and os.path.exists(b_dest):
|
||||
backupdest = module.backup_local(dest)
|
||||
write_changes(module, b_lines, dest)
|
||||
|
||||
if module.check_mode and not os.path.exists(b_dest):
|
||||
module.exit_json(changed=changed, msg=msg, backup=backupdest, diff=diff)
|
||||
@@ -426,38 +441,39 @@ def absent(module, dest, regexp, line, backup):
|
||||
'before_header': '%s (content)' % dest,
|
||||
'after_header': '%s (content)' % dest}
|
||||
|
||||
with open(b_dest, 'rb') as f:
|
||||
b_lines = f.readlines()
|
||||
# NOTE: Avoid opening the same file in this context !
|
||||
with open_locked(dest, module.check_mode) as fd:
|
||||
b_lines = fd.readlines()
|
||||
|
||||
if module._diff:
|
||||
diff['before'] = to_native(b('').join(b_lines))
|
||||
if module._diff:
|
||||
diff['before'] = to_native(b('').join(b_lines))
|
||||
|
||||
if regexp is not None:
|
||||
bre_c = re.compile(to_bytes(regexp, errors='surrogate_or_strict'))
|
||||
found = []
|
||||
|
||||
b_line = to_bytes(line, errors='surrogate_or_strict')
|
||||
|
||||
def matcher(b_cur_line):
|
||||
if regexp is not None:
|
||||
match_found = bre_c.search(b_cur_line)
|
||||
else:
|
||||
match_found = b_line == b_cur_line.rstrip(b('\r\n'))
|
||||
if match_found:
|
||||
found.append(b_cur_line)
|
||||
return not match_found
|
||||
bre_c = re.compile(to_bytes(regexp, errors='surrogate_or_strict'))
|
||||
found = []
|
||||
|
||||
b_lines = [l for l in b_lines if matcher(l)]
|
||||
changed = len(found) > 0
|
||||
b_line = to_bytes(line, errors='surrogate_or_strict')
|
||||
|
||||
if module._diff:
|
||||
diff['after'] = to_native(b('').join(b_lines))
|
||||
def matcher(b_cur_line):
|
||||
if regexp is not None:
|
||||
match_found = bre_c.search(b_cur_line)
|
||||
else:
|
||||
match_found = b_line == b_cur_line.rstrip(b('\r\n'))
|
||||
if match_found:
|
||||
found.append(b_cur_line)
|
||||
return not match_found
|
||||
|
||||
backupdest = ""
|
||||
if changed and not module.check_mode:
|
||||
if backup:
|
||||
backupdest = module.backup_local(dest)
|
||||
write_changes(module, b_lines, dest)
|
||||
b_lines = [l for l in b_lines if matcher(l)]
|
||||
changed = len(found) > 0
|
||||
|
||||
if module._diff:
|
||||
diff['after'] = to_native(b('').join(b_lines))
|
||||
|
||||
backupdest = ""
|
||||
if changed and not module.check_mode:
|
||||
if backup:
|
||||
backupdest = module.backup_local(dest)
|
||||
write_changes(module, b_lines, dest)
|
||||
|
||||
if changed:
|
||||
msg = "%s line(s) removed" % len(found)
|
||||
|
||||
@@ -84,7 +84,6 @@ import re
|
||||
import tempfile
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
from ansible.module_utils.common.file import FileLock
|
||||
from ansible.module_utils._text import to_bytes, to_native
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user