monit: fix module detection of monitored process state (#1107) (#1163)

* refactor and test

* require version >= 5.21.0

Prior to this version the status output was different

* python version compatability

* use exception classes from utils

* modify monit to use 'status' output instead of 'summary' output

The summary output is a fixed width table which truncates the
contents and prevents us from parsing the actual status of the
program.

* add integration tests + fixes

* remove unused handlers in monit integration test

* fix lint

* add '__metaclass__ = type' to integration python files

* raise AttributeError

* simplify status

* lint: add type to parameter docs

* remove lint ignore

* move monit process config into main file

* specify path to monit PID file

* set config location based on os_family

* create required directories

* update aliases to set group and skips

* add changelog

* add author

* add types to docs

* add EPEL repo

* custom vars for centos-6

* uninstall EPEL

* support older versions

* wait for status to change before exiting

* use 'validate' to force status updates

* handle 'execution failed'

* better status output for errors

* add more context to failure + standardize

* don't check rc for validate

* legacy string format support

* add integration test for 'reloaded' and 'present'

* don't wait after reload

* lint

* Revert "uninstall EPEL"

This reverts commit 4d548718d0.

* make 'present' more robust

* Apply suggestions from code review

Co-authored-by: Andrew Klychkov <aaklychkov@mail.ru>

* add license header

* drop daemon.py and use python-daemon instead

* skip python2.6 which is not supported by python-daemon

* refactor test tasks for reuse

* cleanup files after test

* lint

* start process before enabling monit

This shouldn't be necessary but I'm adding it in the hopes
it will make tests more robust.

* retry task

* attempt to rescue the task on failure

* fix indentation

* ignore check if rescue ran

* restart monit instead of reload

Co-authored-by: Andrew Klychkov <aaklychkov@mail.ru>
(cherry picked from commit 8de1c0c205)

Co-authored-by: Simon Kelly <skelly@dimagi.com>

tests/integration/targets/monit/tasks/check_state.yml

@@ -0,0 +1,20 @@
+- name: "{{ reason }} ('up')"
+  command: "curl -sf http://localhost:8082/hello"
+  args:
+    warn: false
+  when: service_state == 'up'
+  register: curl_result
+  until: not curl_result.failed
+  retries: 5
+  delay: 1
+
+- name: "{{ reason }} ('down')"
+  command: "curl -sf http://localhost:8082/hello"
+  args:
+    warn: false
+  register: curl_result
+  failed_when: curl_result == 0
+  when: service_state == 'down'
+  until: not curl_result.failed
+  retries: 5
+  delay: 1

tests/integration/targets/monit/tasks/main.yml

@@ -0,0 +1,78 @@
+####################################################################
+# WARNING: These are designed specifically for Ansible tests       #
+# and should not be used as examples of how to write Ansible roles #
+####################################################################
+
+- block:
+  - name: Install EPEL repository (RHEL only)
+    include_role:
+      name: setup_epel
+
+  - name: create required directories
+    become: yes
+    file:
+      path: "{{ item }}"
+      state: directory
+    loop:
+      - /var/lib/monit
+      - /var/run/monit
+      - "{{ process_root }}"
+
+  - name: install monit
+    become: yes
+    package:
+      name: monit
+      state: present
+
+  - include_vars: '{{ item }}'
+    with_first_found:
+      - files:
+          - "{{ ansible_facts.distribution }}-{{ ansible_facts.distribution_major_version }}.yml"
+          - '{{ ansible_os_family }}.yml'
+          - 'defaults.yml'
+
+  - name: monit config
+    become: yes
+    template:
+      src: "monitrc.j2"
+      dest: "{{ monitrc }}"
+
+  - name: copy process file
+    become: yes
+    copy:
+      src: httpd_echo.py
+      dest: "{{ process_file }}"
+
+  - name: install dependencies
+    pip:
+      name: "{{ item }}"
+      virtualenv: "{{ process_venv }}"
+    loop:
+      - setuptools==44
+      - python-daemon
+
+  - name: restart monit
+    become: yes
+    service:
+      name: monit
+      state: restarted
+
+  - include_tasks: test.yml
+
+  always:
+  - name: stop monit
+    become: yes
+    service:
+      name: monit
+      state: stopped
+
+  - name: uninstall monit
+    become: yes
+    package:
+      name: monit
+      state: absent
+
+  - name: remove process files
+    file:
+      path: "{{ process_root }}"
+      state: absent

tests/integration/targets/monit/tasks/test.yml

@@ -0,0 +1,28 @@
+# order is important
+- import_tasks: test_reload_present.yml
+
+- import_tasks: test_state.yml
+  vars:
+    state: stopped
+    initial_state: up
+    expected_state: down
+
+- import_tasks: test_state.yml
+  vars:
+    state: started
+    initial_state: down
+    expected_state: up
+
+- import_tasks: test_state.yml
+  vars:
+    state: unmonitored
+    initial_state: up
+    expected_state: down
+
+- import_tasks: test_state.yml
+  vars:
+    state: monitored
+    initial_state: down
+    expected_state: up
+
+- import_tasks: test_errors.yml

tests/integration/targets/monit/tasks/test_errors.yml

@@ -0,0 +1,6 @@
+- name: Check an error occurs when wrong process name is used
+  monit:
+    name: missing
+    state: started
+  register: result
+  failed_when: result is not skip and (result is success or result is not failed)

tests/integration/targets/monit/tasks/test_reload_present.yml

@@ -0,0 +1,60 @@
+- name: reload monit when process is missing
+  monit:
+    name: httpd_echo
+    state: reloaded
+  register: result
+
+- name: check that state is changed
+  assert:
+    that:
+      - result is success
+      - result is changed
+
+- name: test process not present
+  monit:
+    name: httpd_echo
+    state: present
+    timeout: 5
+  register: result
+  failed_when: result is not skip and result is success
+
+- name: test monitor missing process
+  monit:
+    name: httpd_echo
+    state: monitored
+  register: result
+  failed_when: result is not skip and result is success
+
+- name: start process
+  shell: "{{ process_run_cmd }}"
+
+- import_tasks: check_state.yml
+  vars:
+    reason: verify service running
+    service_state: "up"
+
+- name: add process config
+  blockinfile:
+    path: "{{ monitrc }}"
+    block: |
+      check process httpd_echo with matching "httpd_echo"
+          start program = "{{ process_run_cmd }}"
+          stop program = "/bin/sh -c 'kill `pgrep -f httpd_echo`'"
+          if failed host localhost port 8082 then restart
+
+- name: restart monit
+  service:
+    name: monit
+    state: restarted
+
+- name: test process present again
+  monit:
+    name: httpd_echo
+    state: present
+  register: result
+
+- name: check that state is unchanged
+  assert:
+    that:
+      - result is success
+      - result is not changed

tests/integration/targets/monit/tasks/test_state.yml

@@ -0,0 +1,33 @@
+- import_tasks: check_state.yml
+  vars:
+    reason: verify initial service state
+    service_state: "{{ initial_state }}"
+
+- name: change httpd_echo process state to {{ state }}
+  monit:
+    name: httpd_echo
+    state: "{{ state }}"
+  register: result
+
+- name: check that state changed
+  assert:
+    that:
+      - result is success
+      - result is changed
+
+- import_tasks: check_state.yml
+  vars:
+    reason: check service state after action
+    service_state: "{{ expected_state }}"
+
+- name: try change state again to {{ state }}
+  monit:
+    name: httpd_echo
+    state: "{{ state }}"
+  register: result
+
+- name: check that state is not changed
+  assert:
+    that:
+      - result is success
+      - result is not changed