RDS inventory plugin (#41919)

Comments out uses of rds_instance in the integration tests and replace with AWS CLI until rds_instance has been merged

test/integration/targets/inventory_aws_rds/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+unsupported

test/integration/targets/inventory_aws_rds/playbooks/create_inventory_config.yml

@@ -0,0 +1,11 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  vars:
+    template_name: "../templates/{{ template | default('inventory.yml') }}"
+  tasks:
+    - name: write inventory config file
+      copy:
+        dest: ../test.aws_rds.yml
+        content: "{{ lookup('template', template_name) }}"

test/integration/targets/inventory_aws_rds/playbooks/empty_inventory_config.yml

@@ -0,0 +1,9 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: write inventory config file
+      copy:
+        dest: ../test.aws_rds.yml
+        content: ""

test/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml

@@ -0,0 +1,76 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+
+    - block:
+        - set_fact:
+            instance_id: 'rds-mariadb-{{ resource_prefix }}'
+
+        - name: assert group was populated with inventory but is empty
+          assert:
+            that:
+              - "'aws_rds' in groups"
+              - "not groups.aws_rds"
+
+        # Create new host, add it to inventory and then terminate it without updating the cache
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: set connection information for all tasks
+        #  set_fact:
+        #    aws_connection_info: &aws_connection_info
+        #      aws_access_key: '{{ aws_access_key }}'
+        #      aws_secret_key: '{{ aws_secret_key }}'
+        #      security_token: '{{ security_token }}'
+        #      region: '{{ aws_region }}'
+        #  no_log: yes
+
+        - name: Use AWS CLI to create an RDS DB instance
+          command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'"
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: create minimal mariadb instance in default VPC and default subnet group
+        #  rds_instance:
+        #    state: present
+        #    engine: mariadb
+        #    instance_class: db.t2.micro
+        #    storage: 20
+        #    instance_id: 'rds-mariadb-{{ resource_prefix }}'
+        #    master_username: 'ansible-test-user'
+        #    master_password: 'password-{{ resource_prefix }}'
+        #    <<: *aws_connection_info
+        #  register: setup_instance
+
+        - meta: refresh_inventory
+
+        - assert:
+            that:
+              - groups.aws_rds
+
+      always:
+
+        - name: Use AWS CLI to delete the DB instance
+          command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
+          ignore_errors: True
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: remove mariadb instance
+        #  rds_instance:
+        #    state: absent
+        #    engine: mariadb
+        #    skip_final_snapshot: yes
+        #    instance_id: ansible-rds-mariadb-example
+        #    <<: *aws_connection_info
+        #  ignore_errors: yes
+        #  when: setup_instance is defined

test/integration/targets/inventory_aws_rds/playbooks/test_invalid_aws_rds_inventory_config.yml

@@ -0,0 +1,9 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: assert inventory was not populated by aws_rds inventory plugin
+      assert:
+        that:
+          - "'aws_rds' not in groups"

test/integration/targets/inventory_aws_rds/playbooks/test_inventory_cache.yml

@@ -0,0 +1,18 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: assert cache was used to populate inventory
+      assert:
+        that:
+          - "'aws_rds' in groups"
+          - "groups.aws_rds | length == 1"
+
+    - meta: refresh_inventory
+
+    - name: assert refresh_inventory updated the cache
+      assert:
+        that:
+          - "'aws_rds' in groups"
+          - "not groups.aws_rds"

test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml

@@ -0,0 +1,106 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+
+    - block:
+
+        - set_fact:
+            instance_id: "{{ resource_prefix }}-mariadb"
+
+        - debug: var=groups
+        - name: assert group was populated with inventory but is empty
+          assert:
+            that:
+              - "'aws_rds' in groups"
+              - "not groups.aws_rds"
+
+        # Create new host, refresh inventory, remove host, refresh inventory
+
+        #- name: set connection information for all tasks
+        #  set_fact:
+        #    aws_connection_info: &aws_connection_info
+        #      aws_access_key: '{{ aws_access_key }}'
+        #      aws_secret_key: '{{ aws_secret_key }}'
+        #      security_token: '{{ security_token }}'
+        #      region: '{{ aws_region }}'
+        #  no_log: yes
+
+        - name: Use AWS CLI to create an RDS DB instance
+          command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'"
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: create minimal mariadb instance in default VPC and default subnet group
+        #  rds_instance:
+        #    state: present
+        #    engine: mariadb
+        #    instance_class: db.t2.micro
+        #    storage: 20
+        #    instance_id: 'rds-mariadb-{{ resource_prefix }}'
+        #    master_username: 'ansible-test-user'
+        #    master_password: 'password-{{ resource_prefix }}'
+        #    <<: *aws_connection_info
+        #  register: setup_instance
+
+        - meta: refresh_inventory
+
+        - name: assert group was populated with inventory and is no longer empty
+          assert:
+            that:
+              - "'aws_rds' in groups"
+              - "groups.aws_rds | length == 1"
+              - "groups.aws_rds.0 == '{{ instance_id }}'"
+
+        - name: Use AWS CLI to delete the DB instance
+          command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
+          ignore_errors: True
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: remove mariadb instance
+        #  rds_instance:
+        #    state: absent
+        #    engine: mariadb
+        #    skip_final_snapshot: yes
+        #    instance_id: ansible-rds-mariadb-example
+        #    <<: *aws_connection_info
+
+        - meta: refresh_inventory
+
+        - name: assert group was populated with inventory but is empty
+          assert:
+            that:
+              - "'aws_rds' in groups"
+              - "not groups.aws_rds"
+
+      always:
+
+        - name: Use AWS CLI to delete the DB instance
+          command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
+          ignore_errors: True
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: remove mariadb instance
+        #  rds_instance:
+        #    state: absent
+        #    engine: mariadb
+        #    skip_final_snapshot: yes
+        #    instance_id: ansible-rds-mariadb-example
+        #    <<: *aws_connection_info
+        #  ignore_errors: yes
+        #  when: setup_instance is defined

test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml

@@ -0,0 +1,76 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+
+    - block:
+
+        - set_fact:
+            instance_id: "{{ resource_prefix }}constructed"
+
+        # Create new host, refresh inventory
+
+        #- name: set connection information for all tasks
+        #  set_fact:
+        #    aws_connection_info: &aws_connection_info
+        #      aws_access_key: '{{ aws_access_key }}'
+        #      aws_secret_key: '{{ aws_secret_key }}'
+        #      security_token: '{{ security_token }}'
+        #      region: '{{ aws_region }}'
+        #  no_log: yes
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: create minimal mariadb instance in default VPC and default subnet group
+        #  rds_instance:
+        #    state: present
+        #    engine: mariadb
+        #    instance_class: db.t2.micro
+        #    storage: 20
+        #    instance_id: 'rds-mariadb-{{ resource_prefix }}'
+        #    master_username: 'ansible-test-user'
+        #    master_password: 'password-{{ resource_prefix }}'
+        #    <<: *aws_connection_info
+        #  register: setup_instance
+
+        - name: Use AWS CLI to create an RDS DB instance
+          command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser' --tags Key='workload_type',Value='other'"
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        - meta: refresh_inventory
+        - debug: var=groups
+
+        - name: assert the keyed groups from constructed config were added to inventory
+          assert:
+            that:
+              # There are 6 groups: all, ungrouped, aws_rds, tag keyed group, engine keyed group, parameter group keyed group
+              - "groups | length == 6"
+              - groups.tag_workload_type_other
+              - groups.rds_mariadb
+              - groups.rds_parameter_group_default_mariadb10_0
+
+      always:
+
+        - name: Use AWS CLI to delete the DB instance
+          command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
+          ignore_errors: True
+          environment:
+            AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+            AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+            AWS_SESSION_TOKEN: "{{ security_token }}"
+            AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+        # TODO: Uncomment once rds_instance has been added
+        #- name: remove mariadb instance
+        #  rds_instance:
+        #    state: absent
+        #    engine: mariadb
+        #    skip_final_snapshot: yes
+        #    instance_id: ansible-rds-mariadb-example
+        #    <<: *aws_connection_info
+        #  ignore_errors: yes
+        #  when: setup_instance is defined

test/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml

@@ -0,0 +1,94 @@
+- name: test updating inventory
+  block:
+    - set_fact:
+        instance_id: "{{ resource_prefix }}update"
+
+    - name: assert group was populated with inventory but is empty
+      assert:
+        that:
+          - "'aws_rds' in groups"
+          - "not groups.aws_rds"
+
+    #- name: set connection information for all tasks
+    #  set_fact:
+    #    aws_connection_info: &aws_connection_info
+    #      aws_access_key: "{{ aws_access_key }}"
+    #      aws_secret_key: "{{ aws_secret_key }}"
+    #      security_token: "{{ security_token }}"
+    #      region: "{{ aws_region }}"
+    #  no_log: yes
+
+    #- name: create minimal mariadb instance in default VPC and default subnet group
+    #  rds_instance:
+    #    state: present
+    #    engine: mariadb
+    #    instance_class: db.t2.micro
+    #    storage: 20
+    #    instance_id: 'rds-mariadb-{{ resource_prefix }}'
+    #    master_username: 'ansible-test-user'
+    #    master_password: 'password-{{ resource_prefix }}'
+    #    <<: *aws_connection_info
+    #  register: setup_instance
+
+    - name: Use AWS CLI to create an RDS DB instance
+      command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+    - meta: refresh_inventory
+
+    - name: assert group was populated with inventory and is no longer empty
+      assert:
+        that:
+          - "'aws_rds' in groups"
+          - "groups.aws_rds | length == 1"
+          - "groups.aws_rds.0 == '{{ resource_prefix }}'"
+
+    #- name: remove mariadb instance
+    #  rds_instance:
+    #    state: absent
+    #    engine: mariadb
+    #    skip_final_snapshot: yes
+    #    instance_id: ansible-rds-mariadb-example
+    #    <<: *aws_connection_info
+
+    - name: Use AWS CLI to delete the DB instance
+      command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
+      ignore_errors: True
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+    - meta: refresh_inventory
+
+    - name: assert group was populated with inventory but is empty
+      assert:
+        that:
+          - "'aws_rds' in groups"
+          - "not groups.aws_rds"
+
+  always:
+
+    - name: Use AWS CLI to delete the DB instance
+      command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
+      ignore_errors: True
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+
+    #- name: remove mariadb instance
+    #  rds_instance:
+    #    state: absent
+    #    engine: mariadb
+    #    skip_final_snapshot: yes
+    #    instance_id: ansible-rds-mariadb-example
+    #    <<: *aws_connection_info
+    #  ignore_errors: yes
+    #  when: setup_instance is defined

test/integration/targets/inventory_aws_rds/runme.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# ensure test config is empty
+ansible-playbook playbooks/empty_inventory_config.yml "$@"
+
+export ANSIBLE_INVENTORY_ENABLED=aws_rds
+
+# test with default inventory file
+ansible-playbook playbooks/test_invalid_aws_rds_inventory_config.yml "$@"
+
+export ANSIBLE_INVENTORY=test.aws_rds.yml
+
+# test empty inventory config
+ansible-playbook playbooks/test_invalid_aws_rds_inventory_config.yml "$@"
+
+# generate inventory config and test using it
+ansible-playbook playbooks/create_inventory_config.yml -e @../../integration_config.yml "$@"
+ansible-playbook playbooks/test_populating_inventory.yml -e @../../integration_config.yml "$@"
+
+# generate inventory config with caching and test using it
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_cache.yml' @../../integration_config.yml" "$@"
+ansible-playbook playbooks/populate_cache.yml -e @../../integration_config.yml "$@"
+ansible-playbook playbooks/test_inventory_cache.yml "$@"
+
+# remove inventory cache
+rm -r aws_rds_cache_dir/
+
+# generate inventory config with constructed features and test using it
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_constructed.yml' @../../integration_config.yml" "$@"
+ansible-playbook playbooks/test_populating_inventory_with_constructed.yml -e @../../integration_config.yml "$@"
+
+# cleanup inventory config
+ansible-playbook playbooks/empty_inventory_config.yml "$@"

test/integration/targets/inventory_aws_rds/templates/inventory.yml

@@ -0,0 +1,6 @@
+plugin: aws_rds
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+aws_security_token: '{{ security_token }}'
+regions:
+  - '{{ aws_region }}'

test/integration/targets/inventory_aws_rds/templates/inventory_with_cache.yml

@@ -0,0 +1,9 @@
+plugin: aws_rds
+cache: True
+cache_plugin: jsonfile
+cache_connection: aws_rds_cache_dir
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+aws_security_token: '{{ security_token }}'
+regions:
+  - '{{ aws_region }}'

test/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.yml

@@ -0,0 +1,13 @@
+plugin: aws_rds
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+aws_security_token: '{{ security_token }}'
+regions:
+  - '{{ aws_region }}'
+keyed_groups:
+  - key: 'db_parameter_groups|json_query("[].db_parameter_group_name")'
+    prefix: rds_parameter_group
+  - key: tags
+    prefix: tag
+  - key: engine
+    prefix: rds