Fix inventory plugin cache + add tests (#38229)

* Fix setting the cache when refresh_cache or --flush-cache are used

* Use jsonify function that handles datetime objects in jsonfile cache plugin

* Don't access self._options directly

* Add initial integration tests for aws_ec2 inventory plugin

* Add CI alias

* Fix and add a few more unit tests

* Add integration tests for constructed

* Fix typo

* Use inventory config templates

* Collect all instances that are not terminated by default

* Create separate playbook for setting up the VPC, subnet, security group, and finding an image for the host

Create a separate playbook for removing the resources

* Allow easier grouping by region and add an example

* use a unified json encode/decode that can handle unsafe and vault

test/integration/targets/aws_ec2_inventory/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+posix/ci/cloud/group4/aws

test/integration/targets/aws_ec2_inventory/playbooks/create_inventory_config.yml

@@ -0,0 +1,11 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  vars:
+    template_name: "../templates/{{ template | default('inventory.yml') }}"
+  tasks:
+    - name: write inventory config file
+      copy:
+        dest: ../test.aws_ec2.yml
+        content: "{{ lookup('template', template_name) }}"

test/integration/targets/aws_ec2_inventory/playbooks/empty_inventory_config.yml

@@ -0,0 +1,9 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: write inventory config file
+      copy:
+        dest: ../test.aws_ec2.yml
+        content: ""

test/integration/targets/aws_ec2_inventory/playbooks/populate_cache.yml

@@ -0,0 +1,63 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+
+    - block:
+
+        # Create VPC, subnet, security group, and find image_id to create instance
+
+        - include_tasks: setup.yml
+
+        - name: assert group was populated with inventory but is empty
+          assert:
+            that:
+              - "'aws_ec2' in groups"
+              - "not groups.aws_ec2"
+
+        # Create new host, add it to inventory and then terminate it without updating the cache
+
+        - name: set connection information for all tasks
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: '{{ aws_access_key }}'
+              aws_secret_key: '{{ aws_secret_key }}'
+              security_token: '{{ security_token }}'
+              region: '{{ aws_region }}'
+          no_log: yes
+
+        - name: create a new host
+          ec2:
+            image: '{{ image_id }}'
+            exact_count: 1
+            count_tag:
+              Name: '{{ resource_prefix }}'
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            instance_type: t2.micro
+            wait: yes
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+            <<: *aws_connection_info
+          register: setup_instance
+
+        - meta: refresh_inventory
+
+      always:
+
+        - name: remove setup ec2 instance
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+            <<: *aws_connection_info
+          ignore_errors: yes
+          when: setup_instance is defined
+
+        - include_tasks: tear_down.yml

test/integration/targets/aws_ec2_inventory/playbooks/setup.yml

@@ -0,0 +1,62 @@
+- name: set connection information for all tasks
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: '{{ aws_access_key }}'
+      aws_secret_key: '{{ aws_secret_key }}'
+      security_token: '{{ security_token }}'
+      region: '{{ aws_region }}'
+    no_log: yes
+
+- name: get image ID to create an instance
+  ec2_ami_facts:
+    filters:
+      architecture: x86_64
+      owner-id: '125523088429'
+      virtualization-type: hvm
+      root-device-type: ebs
+      name: 'Fedora-Atomic-27*'
+    <<: *aws_connection_info
+  register: fedora_images
+
+- set_fact:
+    image_id: '{{ fedora_images.images.0.image_id }}'
+
+- name: create a VPC to work in
+  ec2_vpc_net:
+    cidr_block: 10.10.0.0/24
+    state: present
+    name: '{{ resource_prefix }}_setup'
+    resource_tags:
+      Name: '{{ resource_prefix }}_setup'
+    <<: *aws_connection_info
+  register: setup_vpc
+
+- set_fact:
+    vpc_id: '{{ setup_vpc.vpc.id }}'
+
+- name: create a subnet to use for creating an ec2 instance
+  ec2_vpc_subnet:
+    az: '{{ aws_region }}a'
+    tags: '{{ resource_prefix }}_setup'
+    vpc_id: '{{ setup_vpc.vpc.id }}'
+    cidr: 10.10.0.0/24
+    state: present
+    resource_tags:
+      Name: '{{ resource_prefix }}_setup'
+    <<: *aws_connection_info
+  register: setup_subnet
+
+- set_fact:
+    subnet_id: '{{ setup_subnet.subnet.id }}'
+
+- name: create a security group to use for creating an ec2 instance
+  ec2_group:
+    name: '{{ resource_prefix }}_setup'
+    description: 'created by Ansible integration tests'
+    state: present
+    vpc_id: '{{ setup_vpc.vpc.id }}'
+    <<: *aws_connection_info
+  register: setup_sg
+
+- set_fact:
+    sg_id: '{{ setup_sg.group_id }}'

test/integration/targets/aws_ec2_inventory/playbooks/tear_down.yml

@@ -0,0 +1,39 @@
+- name: set connection information for all tasks
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: '{{ aws_access_key }}'
+      aws_secret_key: '{{ aws_secret_key }}'
+      security_token: '{{ security_token }}'
+      region: '{{ aws_region }}'
+    no_log: yes
+
+- name: remove setup security group
+  ec2_group:
+    name: '{{ resource_prefix }}_setup'
+    description: 'created by Ansible integration tests'
+    state: absent
+    vpc_id: '{{ vpc_id }}'
+    <<: *aws_connection_info
+  ignore_errors: yes
+
+- name: remove setup subnet
+  ec2_vpc_subnet:
+    az: '{{ aws_region }}a'
+    tags: '{{ resource_prefix }}_setup'
+    vpc_id: '{{ vpc_id }}'
+    cidr: 10.10.0.0/24
+    state: absent
+    resource_tags:
+      Name: '{{ resource_prefix }}_setup'
+    <<: *aws_connection_info
+  ignore_errors: yes
+
+- name: remove setup VPC
+  ec2_vpc_net:
+    cidr_block: 10.10.0.0/24
+    state: absent
+    name: '{{ resource_prefix }}_setup'
+    resource_tags:
+      Name: '{{ resource_prefix }}_setup'
+    <<: *aws_connection_info
+  ignore_errors: yes

test/integration/targets/aws_ec2_inventory/playbooks/test_invalid_aws_ec2_inventory_config.yml

@@ -0,0 +1,9 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: assert inventory was not populated by aws_ec2 inventory plugin
+      assert:
+        that:
+          - "'aws_ec2' not in groups"

test/integration/targets/aws_ec2_inventory/playbooks/test_inventory_cache.yml

@@ -0,0 +1,18 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: assert cache was used to populate inventory
+      assert:
+        that:
+          - "'aws_ec2' in groups"
+          - "groups.aws_ec2 | length == 1"
+
+    - meta: refresh_inventory
+
+    - name: assert refresh_inventory updated the cache
+      assert:
+        that:
+          - "'aws_ec2' in groups"
+          - "not groups.aws_ec2"

test/integration/targets/aws_ec2_inventory/playbooks/test_populating_inventory.yml

@@ -0,0 +1,90 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+
+    - block:
+
+        # Create VPC, subnet, security group, and find image_id to create instance
+
+        - include_tasks: setup.yml
+
+        - name: assert group was populated with inventory but is empty
+          assert:
+            that:
+              - "'aws_ec2' in groups"
+              - "not groups.aws_ec2"
+
+        # Create new host, refresh inventory, remove host, refresh inventory
+
+        - name: set connection information for all tasks
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: '{{ aws_access_key }}'
+              aws_secret_key: '{{ aws_secret_key }}'
+              security_token: '{{ security_token }}'
+              region: '{{ aws_region }}'
+          no_log: yes
+
+        - name: create a new host
+          ec2:
+            image: '{{ image_id }}'
+            exact_count: 1
+            count_tag:
+              Name: '{{ resource_prefix }}'
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            instance_type: t2.micro
+            wait: yes
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+            <<: *aws_connection_info
+          register: setup_instance
+
+        - meta: refresh_inventory
+
+        - name: assert group was populated with inventory and is no longer empty
+          assert:
+            that:
+              - "'aws_ec2' in groups"
+              - "groups.aws_ec2 | length == 1"
+              - "groups.aws_ec2.0 == '{{ resource_prefix }}'"
+
+        - name: remove setup ec2 instance
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+            <<: *aws_connection_info
+
+        - meta: refresh_inventory
+
+        - name: assert group was populated with inventory but is empty
+          assert:
+            that:
+              - "'aws_ec2' in groups"
+              - "not groups.aws_ec2"
+
+      always:
+
+        - name: remove setup ec2 instance
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+            <<: *aws_connection_info
+          ignore_errors: yes
+          when: setup_instance is defined
+
+        - include_tasks: tear_down.yml

test/integration/targets/aws_ec2_inventory/playbooks/test_populating_inventory_with_constructed.yml

@@ -0,0 +1,78 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  tasks:
+
+    - block:
+
+        # Create VPC, subnet, security group, and find image_id to create instance
+
+        - include_tasks: setup.yml
+
+        # Create new host, refresh inventory
+
+        - name: set connection information for all tasks
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: '{{ aws_access_key }}'
+              aws_secret_key: '{{ aws_secret_key }}'
+              security_token: '{{ security_token }}'
+              region: '{{ aws_region }}'
+          no_log: yes
+
+        - name: create a new host
+          ec2:
+            image: '{{ image_id }}'
+            exact_count: 1
+            count_tag:
+              Name: '{{ resource_prefix }}'
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+              tag1: value1
+              tag2: value2
+            instance_type: t2.micro
+            wait: yes
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+            <<: *aws_connection_info
+          register: setup_instance
+
+        - meta: refresh_inventory
+
+        - name: register the keyed sg group name
+          set_fact:
+            sg_group_name: "security_groups_{{ sg_id | replace('-', '_') }}"
+
+        - name: register one of the keyed tag groups name
+          set_fact:
+            tag_group_name: "tag_Name_{{ resource_prefix | replace('-', '_') }}"
+
+        - name: assert the keyed groups and groups from constructed config were added to inventory and composite var added to hostvars
+          assert:
+            that:
+              # There are 9 groups: all, ungrouped, aws_ec2, sg keyed group, 3 tag keyed group (one per tag), arch keyed group, constructed group
+              - "groups | length == 9"
+              - "groups[tag_group_name] | length == 1"
+              - "groups[sg_group_name] | length == 1"
+              - "groups.arch_x86_64 | length == 1"
+              - "groups.tag_with_name_key | length == 1"
+              - vars.hostvars[groups.aws_ec2.0]['test_compose_var_sum'] == 'value1value2'
+
+      always:
+
+        - name: remove setup ec2 instance
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: "{{ sg_id }}"
+            vpc_subnet_id: "{{ subnet_id }}"
+            <<: *aws_connection_info
+          ignore_errors: yes
+          when: setup_instance is defined
+
+        - include_tasks: tear_down.yml

test/integration/targets/aws_ec2_inventory/playbooks/test_refresh_inventory.yml

@@ -0,0 +1,74 @@
+- name: test updating inventory
+  block:
+    - name: assert group was populated with inventory but is empty
+      assert:
+        that:
+          - "'aws_ec2' in groups"
+          - "not groups.aws_ec2"
+
+    - name: set connection information for all tasks
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: yes
+
+    - name: create a new host
+      ec2:
+        image: "{{ images[aws_region] }}"
+        exact_count: 1
+        count_tag:
+          Name: '{{ resource_prefix }}'
+        instance_tags:
+          Name: '{{ resource_prefix }}'
+        instance_type: t2.micro
+        wait: yes
+        group_id: '{{ setup_sg.group_id }}'
+        vpc_subnet_id: '{{ setup_subnet.subnet.id }}'
+        <<: *aws_connection_info
+      register: setup_instance
+
+    - meta: refresh_inventory
+
+    - name: assert group was populated with inventory and is no longer empty
+      assert:
+        that:
+          - "'aws_ec2' in groups"
+          - "groups.aws_ec2 | length == 1"
+          - "groups.aws_ec2.0 == '{{ resource_prefix }}'"
+
+    - name: remove setup ec2 instance
+      ec2:
+        instance_type: t2.micro
+        instance_ids: '{{ setup_instance.instance_ids }}'
+        state: absent
+        wait: yes
+        instance_tags:
+          Name: '{{ resource_prefix }}'
+        group_id: '{{ setup_sg.group_id }}'
+        vpc_subnet_id: '{{ setup_subnet.subnet.id }}'
+        <<: *aws_connection_info
+
+    - meta: refresh_inventory
+
+    - name: assert group was populated with inventory but is empty
+      assert:
+        that:
+          - "'aws_ec2' in groups"
+          - "not groups.aws_ec2"
+
+  always:
+    - name: remove setup ec2 instance
+      ec2:
+        instance_type: t2.micro
+        instance_ids: '{{ setup_instance.instance_ids }}'
+        state: absent
+        wait: yes
+        instance_tags:
+          Name: '{{ resource_prefix }}'
+        group_id: '{{ setup_sg.group_id }}'
+        vpc_subnet_id: '{{ setup_subnet.subnet.id }}'
+        <<: *aws_connection_info
+      ignore_errors: yes

test/integration/targets/aws_ec2_inventory/runme.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# ensure test config is empty
+ansible-playbook playbooks/empty_inventory_config.yml "$@"
+
+export ANSIBLE_INVENTORY_ENABLED=aws_ec2
+
+# test with default inventory file
+ansible-playbook playbooks/test_invalid_aws_ec2_inventory_config.yml "$@"
+
+export ANSIBLE_INVENTORY=test.aws_ec2.yml
+
+# test empty inventory config
+ansible-playbook playbooks/test_invalid_aws_ec2_inventory_config.yml "$@"
+
+# generate inventory config and test using it
+ansible-playbook playbooks/create_inventory_config.yml -e @../../integration_config.yml "$@"
+ansible-playbook playbooks/test_populating_inventory.yml -e @../../integration_config.yml "$@"
+
+# generate inventory config with caching and test using it
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_cache.yml' @../../integration_config.yml" "$@"
+ansible-playbook playbooks/populate_cache.yml -e @../../integration_config.yml "$@"
+ansible-playbook playbooks/test_inventory_cache.yml "$@"
+
+# remove inventory cache
+rm -r aws_ec2_cache_dir/
+
+# generate inventory config with constructed features and test using it
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_constructed.yml' @../../integration_config.yml" "$@"
+ansible-playbook playbooks/test_populating_inventory_with_constructed.yml -e @../../integration_config.yml "$@"
+
+# cleanup inventory config
+ansible-playbook playbooks/empty_inventory_config.yml "$@"

test/integration/targets/aws_ec2_inventory/templates/inventory.yml

@@ -0,0 +1,12 @@
+plugin: aws_ec2
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+aws_security_token: '{{ security_token }}'
+regions:
+  - '{{ aws_region }}'
+filters:
+  tag:Name:
+    - '{{ resource_prefix }}'
+hostnames:
+  - tag:Name
+  - dns-name

test/integration/targets/aws_ec2_inventory/templates/inventory_with_cache.yml

@@ -0,0 +1,12 @@
+plugin: aws_ec2
+cache: True
+cache_plugin: jsonfile
+cache_connection: aws_ec2_cache_dir
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+aws_security_token: '{{ security_token }}'
+regions:
+  - '{{ aws_region }}'
+filters:
+  tag:Name:
+    - '{{ resource_prefix }}'

test/integration/targets/aws_ec2_inventory/templates/inventory_with_constructed.yml

@@ -0,0 +1,20 @@
+plugin: aws_ec2
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+aws_security_token: '{{ security_token }}'
+regions:
+  - '{{ aws_region }}'
+filters:
+  tag:Name:
+    - '{{ resource_prefix }}'
+keyed_groups:
+  - key: 'security_groups|json_query("[].group_id")'
+    prefix: 'security_groups'
+  - key: 'tags'
+    prefix: 'tag'
+  - prefix: 'arch'
+    key: "architecture"
+compose:
+  test_compose_var_sum: tags.tag1 + tags.tag2
+groups:
+  tag_with_name_key: "'Name' in (tags | list)"