Use locking for concurrent file access (#52567)

* Use locking for concurrent file access

This implements locking to be used for modules that are used for
concurrent file access, like lineinfile or known_hosts.

* Reinstate lock_timeout

This commit includes:
- New file locking infrastructure for modules
- Enable timeout tests
- Madifications to support concurrency with lineinfile

* Rebase, update changelog and tests

We need to specify ansible_python_interpreter to avoid running interpreter discovery and selecting the incorrect interpreter.

Remove the import of lock in known_hosts since it is not used.

test/integration/targets/file_lock/aliases

@@ -0,0 +1 @@
+shippable/posix/group2

test/integration/targets/file_lock/inventory

@@ -0,0 +1,2 @@
+[lockhosts]
+lockhost[00:99] ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}"

test/integration/targets/file_lock/runme.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_filelock.yml -i inventory --forks 10 --diff -v "$@"
+ansible-playbook test_filelock_timeout.yml -i inventory --diff  -v "$@"

test/integration/targets/file_lock/test_filelock.yml

@@ -0,0 +1,45 @@
+---
+- hosts: lockhosts
+  gather_facts: no
+  vars:
+    lockfile: ~/ansible_testing/lock.test
+  tasks:
+  - name: Remove lockfile
+    file:
+      path: '{{ lockfile }}'
+      state: absent
+    run_once: yes
+
+  - name: Write inventory_hostname to lockfile concurrently
+    lineinfile:
+      path: '{{ lockfile }}'
+      line: '{{ inventory_hostname }}'
+      create: yes
+      state: present
+
+  - debug:
+      msg: File {{ lockfile }} has {{ lines|length }} lines for {{ ansible_play_batch|length }} instances
+    vars:
+      lines: "{{ lookup('file', lockfile).split('\n') }}"
+    run_once: yes
+
+  - name: Assert we get the expected number of lines
+    assert:
+      that:
+      - lines|length == ansible_play_batch|length
+    vars:
+      lines: "{{ lookup('file', lockfile).split('\n') }}"
+    run_once: yes
+
+  - name: Check lockfile for inventory_hostname entries
+    lineinfile:
+      path: '{{ lockfile }}'
+      line: '{{ inventory_hostname }}'
+      state: present
+    register: check_lockfile
+
+  - name: Assert locking results
+    assert:
+      that:
+      - check_lockfile is not changed
+      - check_lockfile is not failed

test/integration/targets/file_lock/test_filelock_timeout.yml

@@ -0,0 +1,63 @@
+---
+- hosts: lockhost00
+  vars:
+    lockfile: ~/ansible_testing/lock_timeout.test
+  gather_facts: no
+  tasks:
+  - name: Remove lockfile
+    file:
+      path: '{{ lockfile }}'
+      state: absent
+    run_once: yes
+
+  - name: Create lockfile
+    lineinfile:
+      line: '{{ inventory_hostname }}'
+      path: '{{ lockfile }}'
+      state: present
+      create: yes
+
+  - name: Lock lockfile with lockf and sleep 20s
+    command: python
+    args:
+      stdin: |
+        import time
+        from ansible.module_utils.common.file import open_locked
+        with open_locked('{{ lockfile | expanduser }}') as fd:
+            time.sleep(20)
+    async: 60
+    poll: 0
+    register: flock_waiter
+
+  - name: Remove inventory_hostname line from lockfile
+    lineinfile:
+      path: '{{ lockfile }}'
+      line: '{{ inventory_hostname }}'
+      state: absent
+    ignore_errors: yes
+    register: rm_line
+
+  - name: Assert that removal of inventory_hostname from lockfile failed
+    assert:
+      that:
+        - rm_line is failed
+
+  - name: Wait for flock job to finish
+    async_status:
+      jid: '{{ flock_waiter.ansible_job_id }}'
+    register: job_result
+    until: job_result.finished
+    retries: 30
+
+  - name: Inventory_hostname in lockfile
+    lineinfile:
+      path: '{{ lockfile }}'
+      line: '{{ inventory_hostname }}'
+      state: present
+    register: check_line
+
+  - name: Assert that lockfile is unchanged
+    assert:
+      that:
+        - check_line is not changed
+        - check_line is not failed