[AWS] ses rule set module for inbound email processing (#42781)

* Add module ses_rule_set for Amazon SES

* Update behaviours and naming to be consistent with other aws_ses_ modules.

* Add global lock around tests using active rule sets to prevent intermittent test failures.

* Fix deletion of rule sets so that we don't inactivate the active rule set
when force deleting an inactive rule set.

test/integration/targets/aws_ses_rule_set/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+shippable/aws/group1

test/integration/targets/aws_ses_rule_set/defaults/main.yaml

@@ -0,0 +1,9 @@
+---
+default_rule_set: "{{ resource_prefix }}-default-rule-set"
+second_rule_set: "{{ resource_prefix }}-second-rule-set"
+
+# See comment in obtain-lock.yaml for definitions of these variables
+max_obtain_lock_attempts: 10
+obtain_lock_delay_seconds: 30
+lock_timeout_seconds: 900
+lock_log_group_prefix: "ansible-testing-locks/aws_ses_rule_set"

test/integration/targets/aws_ses_rule_set/tasks/active-rule-set-tests.yaml

@@ -0,0 +1,349 @@
+---
+# ============================================================
+# These tests all rely on making rule sets active. There can only be
+# a single active rule set so multiple builds must not run these tests
+# in parallel or they will fail intermittently.
+# See the locking block in main.yaml for how this restriction is enforced
+# ============================================================
+
+- name: set up aws connection info
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token }}"
+      region: "{{ aws_region }}"
+  no_log: yes
+
+# ============================================================
+- name: mark rule set active
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+    - name: mark rule set active
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed to active
+      assert:
+        that:
+          - result.changed == True
+          - result.active == True
+    - name: remark rule set active
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is False
+      assert:
+        that:
+          - result.changed == False
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: create rule set active
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed to existing and active
+      assert:
+        that:
+          - result.changed == True
+          - result.active == True
+          - "default_rule_set in result.rule_sets|map(attribute='name')"
+    - name: remark rule set active
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is False
+      assert:
+        that:
+          - result.changed == False
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: mark rule set inactive
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: mark rule set inactive
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: False
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed to inactive
+      assert:
+        that:
+          - result.changed == True
+          - result.active == False
+    - name: remark rule set inactive
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: False
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is False
+      assert:
+        that:
+          - result.changed == False
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: Absent active flag does not change active status
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: recreate rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+      register: result
+    - name: assert not changed and still active
+      assert:
+        that:
+          - result.changed == False
+          - result.active == True
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: Cannot Remove Active Rule Set
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: remove rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+      failed_when: "result.error.code != 'CannotDelete'"
+    - name: assert error code is CannotDelete
+      assert:
+        that:
+          - "result.error.code == 'CannotDelete'"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: Remove Active Rule Set with Force
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: force remove rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed and absent
+      assert:
+        that:
+          - result.changed == True
+          - "default_rule_set not in result.rule_sets|map(attribute='name')"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: Force Remove of Inactive Rule Set does Not Affect Active Rule Set
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: create inactive rule set
+      aws_ses_rule_set:
+        name: "{{ second_rule_set }}"
+        active: False
+        <<: *aws_connection_info
+    - name: force remove inactiave rule set
+      aws_ses_rule_set:
+        name: "{{ second_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed and absent
+      assert:
+        that:
+          - result.changed == True
+          - "second_rule_set not in result.rule_sets|map(attribute='name')"
+    - name: remark active rule set active
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert no change
+      assert:
+        that:
+          - result.changed == False
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ item }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      loop:
+        - "{{ default_rule_set }}"
+        - "{{ second_rule_set }}"
+
+# ============================================================
+- name: mark rule set inactive in check mode
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: mark rule set inactive in check mode
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: False
+        <<: *aws_connection_info
+      register: result
+      check_mode: True
+    - name: assert changed to inactive
+      assert:
+        that:
+          - result.changed == True
+          - result.active == False
+    - name: remark rule set inactive
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: False
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is True since previous inactive was in check mode
+      assert:
+        that:
+          - result.changed == True
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: Cannot Remove Active Rule Set in check mode
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: remove rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+      failed_when: "result.error.code != 'CannotDelete'"
+      check_mode: True
+    - name: assert error code is CannotDelete
+      assert:
+        that:
+          - "result.error.code == 'CannotDelete'"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+
+# ============================================================
+- name: Remove Active Rule Set with Force in check mode
+  block:
+    - name: create active rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+    - name: force remove rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      register: result
+      check_mode: True
+    - name: assert changed and absent
+      assert:
+        that:
+          - result.changed == True
+          - "default_rule_set not in result.rule_sets|map(attribute='name')"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is True since previous removal was in check mode
+      assert:
+        that:
+          - result.changed == True

test/integration/targets/aws_ses_rule_set/tasks/cleanup-lock.yaml

@@ -0,0 +1,15 @@
+---
+# ============================================================
+# Release a lock obtained using obtain-lock.yaml
+# This should be included in the always clause of a block to
+# ensure the lock is released. See obtain-lock.yaml for more
+# details of how the locking works.
+# ============================================================
+
+- cloudwatchlogs_log_group:
+    log_group_name: "{{ lock_attempt_log_group_name }}"
+    state: absent
+    aws_access_key: "{{ aws_access_key }}"
+    aws_secret_key: "{{ aws_secret_key }}"
+    security_token: "{{ security_token }}"
+    region: "{{ aws_region }}"

test/integration/targets/aws_ses_rule_set/tasks/inactive-rule-set-tests.yaml

@@ -0,0 +1,187 @@
+---
+# ============================================================
+# These tests work on rule sets without making them active.
+# so multiple builds can safely run these tests as is normal.
+#
+# DO NOT ADD TESTS THAT RELY ON ACTIVE RULE SETS TO THIS FILE
+#
+# Any test that make rule sets active must be added in
+# active-rule-set-tests.yaml or you will have intermittent failures
+# from multiple builds interacting
+# ============================================================
+
+- name: set up aws connection info
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token }}"
+      region: "{{ aws_region }}"
+  no_log: yes
+
+# ============================================================
+- name: test create rule sets
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed to exists inactive
+      assert:
+        that:
+          - result.changed == True
+          - result.active == False
+          - "default_rule_set in result.rule_sets|map(attribute='name')"
+    - name: recreate rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is False
+      assert:
+        that:
+          - result.changed == False
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+# ============================================================
+- name: Remove No Such Rules Set
+  block:
+    - name: remove ruleset
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+    - name: assert not changed and absent
+      assert:
+        that:
+          - result.changed == False
+          - "default_rule_set not in result.rule_sets|map(attribute='name')"
+# ============================================================
+- name: Remove Inactive Rule Set
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+    - name: remove rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed and removed
+      assert:
+        that:
+          - result.changed == True
+          - "default_rule_set not in result.rule_sets|map(attribute='name')"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+# ============================================================
+- name: test create in check mode
+  block:
+    - name: create rule set in check mode
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+      register: result
+      check_mode: True
+    - name: assert changed inactive and present
+      assert:
+        that:
+          - result.changed == True
+          - result.active == False
+          - "default_rule_set in result.rule_sets|map(attribute='name')"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert nothing to clean up since create was in check mode
+      assert:
+        that:
+          - result.changed == False
+# ============================================================
+- name: mark rule set active in check mode
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+    - name: mark rule set active in check mode
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: True
+        <<: *aws_connection_info
+      register: result
+      check_mode: True
+    - name: assert changed and active
+      assert:
+        that:
+          - result.changed == True
+          - result.active == True
+    # We check the rule set is still inactive rather than making
+    # it active again as that way this test can be run in 
+    # parallel
+    - name: Ensure rule set is inactive
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        active: False
+        <<: *aws_connection_info
+      register: result
+    - name: assert unchanged since activation was in check mode
+      assert:
+        that:
+          - result.changed == False
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+# ============================================================
+- name: Remove Inactive Rule Set in check mode
+  block:
+    - name: create rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        <<: *aws_connection_info
+    - name: remove rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+      check_mode: True
+    - name: assert changed and removed
+      assert:
+        that:
+          - result.changed == True
+          - "default_rule_set not in result.rule_sets|map(attribute='name')"
+  always:
+    - name: cleanup rule set
+      aws_ses_rule_set:
+        name: "{{ default_rule_set }}"
+        state: absent
+        force: True
+        <<: *aws_connection_info
+      register: result
+    - name: assert changed is True since previous removal was in check mode
+      assert:
+        that:
+          - result.changed == True

test/integration/targets/aws_ses_rule_set/tasks/main.yaml

@@ -0,0 +1,36 @@
+---
+- include_tasks: inactive-rule-set-tests.yaml
+
+# ============================================================
+# There can only be a single active rule set, tests that
+# relies on the active state of the rule cannot be run in
+# parallel.
+# To prevent failures due to parallel runs in the integration
+# builds, the below block creates a lock to ensure that only
+# one process will be running these tests in the same region
+# and same AWS account.
+# See obtain-lock.yaml for explanation of how the lock is
+# constructed.
+# ============================================================
+- name: Active Rule Set Tests
+  block:
+    - name: Obtain Lock
+      include_tasks: obtain-lock-wrapper.yaml
+      # Use of loop here is a workaround for lack of support for
+      # do-until loops on includes. See:
+      # https://github.com/ansible/ansible/issues/17098
+      loop: "{{ range(0, max_obtain_lock_attempts, 1)|list }}"
+      loop_control:
+        loop_var: obtain_lock_attempt
+
+      # Because of the above workaround we have to explicitly check
+      # that the lock was obtained
+    - name: Check Obtained Lock
+      assert:
+        msg: "Could not obtain lock after {{ max_obtain_lock_attempts }} attempts."
+        that: won_lock|bool
+
+    - include_tasks: active-rule-set-tests.yaml
+
+  always:
+    - include_tasks: cleanup-lock.yaml

test/integration/targets/aws_ses_rule_set/tasks/obtain-lock-wrapper.yaml

@@ -0,0 +1,26 @@
+# ============================================================
+# Do While loops cannot be used on task includes.
+# See: https://github.com/ansible/ansible/issues/17098
+#
+# So as a workaround we use a regular loop to repeatedly attempt
+# obtaining a lock.
+#
+# For this to work we need to skip the subsequent iterations
+# once we get a lock, and delay between iterations if we
+# did not obtain the lock.
+#
+# This file encapsulates this logic to reduce the spam from
+# skipped tasks in the ansible log.
+# ============================================================
+
+- include_tasks: obtain-lock.yaml
+  # Skip obtaining a lock if we've already succeeded in getting it
+  when: "not won_lock|default(False)|bool"
+
+- name: Lock Retry Delay
+  wait_for:
+    # Add some random jitter to the delay to reduce lock contention
+    timeout: "{{ obtain_lock_delay_seconds + 15|random }}"
+  # Only delay if we're retrying, so skip the delay if we're
+  # on the last attempt or have got the lock
+  when: "obtain_lock_attempt < (max_obtain_lock_attempts - 1) and not won_lock|bool"

test/integration/targets/aws_ses_rule_set/tasks/obtain-lock.yaml

@@ -0,0 +1,126 @@
+# ============================================================
+# This file attempts to obtain a global lock (for a given
+# region / account combination.
+#
+# This makes one attempt to get the lock and will set the
+# won_lock variable to True or False to indicate whether
+# or not we got the lock.
+#
+# It's expected that this will be executed in a retry loop
+# so that if we don't get the lock we delay then try again.
+#
+# This should only be used in a block with cleanup-lock.yaml
+# included in the always clause to ensure the lock is released.
+#
+# There are several variables that control the locking behaviour:
+# * lock_timeout_seconds
+#     How old a lock must be before it's assumed to be an expired
+#     lock that was not cleaned up by the owner. Any locks older
+#     than this will not prevent a lock being obtained and will
+#     be deleted when a new process obtains the lock.
+# * lock_log_group_prefix
+#     The log_group prefix that represents the lock being obtained.
+#     This must be the same across all processes trying to obtain
+#     the lock.
+# * lock_process_id
+#     A unique identifier of this process. Each process that might
+#     attempt to lock the process must have a different identifier.
+#     This defaults to the resource_prefix which is generally
+#     appropriate.
+# * max_obtain_lock_attempts
+#     How many attempts to make to get the lock before giving up
+#     NB: This is actually done in main.yaml
+# * obtain_lock_delay_seconds:
+#     How long to delay after failing to get the lock before
+#     trying again.
+#     NB: This is actually done in obtain-lock-wrapper.yaml
+#
+# The locking here is based around creating cloudwatch log groups.
+# This resource was chosen because:
+#   A) it's free
+#   B) we have a built in grouping concept because of the hierarchy
+#      that allows us to easily group attempts for the same lock
+#   C) the creation time is tracked and returned which gives us
+#      a mechanism for deterministically picking a winner
+#
+# Each lock is represented by a log group prefix. Each attempt
+# to obtain the lock is a log group of the lock_process_id below
+# that prefix.
+#
+# The winning lock is the one with the earliest creation time.
+#
+# To prevent a hanging lock from permanently hanging the build
+# lock attempts older than the lock timeout are ignored and
+# cleaned up by the next process to win the lock.
+# ============================================================
+
+- name: set up aws connection info
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token }}"
+      region: "{{ aws_region }}"
+  no_log: yes
+
+- name: Set lock_attempt_log_group_name
+  set_fact:
+    lock_attempt_log_group_name: "{{ lock_log_group_prefix }}/{{ lock_process_id|default(resource_prefix) }}"
+
+  # Note the overwrite below to ensure that the creation time
+  # is upated. This is important as we calculate expiry relative
+  # the attempt creation.
+  #
+  # Because of this it's imporatnt that we delete the attempt
+  # if we don't get the lock. Otherwise we can get a deadlock
+  # where the stale atttempt from one process wins, but then
+  # because that process updates the creation date it doesn't
+  # consider its self to havewone.
+- name: Create Lock Attempt Log Group
+  cloudwatchlogs_log_group:
+    log_group_name: "{{ lock_attempt_log_group_name }}"
+    state: present
+    overwrite: True
+    <<: *aws_connection_info
+  register: lock_attempt_log_group_result
+
+- name: Get Lock Attempt Lock Groups
+  cloudwatchlogs_log_group_facts:
+    log_group_name: "{{ lock_log_group_prefix }}/"
+    <<: *aws_connection_info
+  register: lock_attempt_log_groups
+
+- name: Calculate Expired Lock Attempt Timestamp
+  set_fact:
+    expired_lock_timestamp: "{{ lock_attempt_log_group_result.creation_time - (lock_timeout_seconds * 1000) }}"
+
+- name: Get Expired and Active Lock Attempts
+  set_fact:
+    expired_lock_attempts: "{{ lock_attempt_log_groups.log_groups|selectattr('creation_time', 'lt', expired_lock_timestamp|int)|list }}"
+    active_lock_attempts: "{{ lock_attempt_log_groups.log_groups|selectattr('creation_time', 'ge', expired_lock_timestamp|int)|list }}"
+
+- name: Pick Winning Lock Attempt
+  set_fact:
+    winning_lock_attempt: "{{ active_lock_attempts|sort(attribute='creation_time')|first }}"
+
+- name: Determine if Won Lock
+  set_fact:
+    won_lock: "{{ winning_lock_attempt.log_group_name == lock_attempt_log_group_name }}"
+
+  # Remove the lock attempt if we didn't get the lock. This prevents
+  # our stale lock attempt blocking another process from getting the lock.
+  # See more detailed comment above Create Lock Attempt Log Group
+- name: Remove Failed Lock Attempt Log Group
+  cloudwatchlogs_log_group:
+    log_group_name: "{{ lock_attempt_log_group_name }}"
+    state: absent
+    <<: *aws_connection_info
+  when: "not won_lock|bool"
+
+- name: Delete Expired Lock Attempts
+  cloudwatchlogs_log_group:
+    log_group_name: "{{ item.log_group_name }}"
+    state: absent
+    <<: *aws_connection_info
+  when: "won_lock|bool"
+  loop: "{{ expired_lock_attempts }}"