parse botocore.endpoint logs into a list of AWS actions (#49312)

* Add an option to parse botocore.endpoint logs for the AWS actions performed during a task

Add a callback to consolidate all AWS actions used by modules

Added some documentation to the AWS guidelines

* Enable aws_resource_actions callback only for AWS tests

* Add script to help generate policies

* Set debug_botocore_endpoint_logs via environment variable for all AWS integration tests

Ensure AWS tests inherit environment

(also remove AWS CLI in aws_rds inventory tests and use the module)
This commit is contained in:
Sloane Hertel
2019-03-18 08:29:03 -05:00
committed by GitHub
parent eb790cd3c6
commit 7da565b3ae
34 changed files with 672 additions and 233 deletions

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- aws_eks

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
tasks:
- name: try and use aws_eks_cluster module

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- ../../cloudformation_stack_set

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- ec2_instance

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing'

View File

@@ -1,4 +1,5 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- ec2_launch_template

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing'
module_defaults:

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- ecs_cluster

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing'

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing'

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing-fnd'

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing-fndf'

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing'

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
vars:
resource_prefix: 'ansible-testing'

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- elb_target

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
tasks:
- name: set up aws connection info

View File

@@ -1,5 +1,6 @@
- hosts: localhost
connection: local
environment: "{{ ansible_test.environment }}"
roles:
- elb_target_facts

View File

@@ -2,6 +2,7 @@
- hosts: 127.0.0.1
connection: local
gather_facts: no
environment: "{{ ansible_test.environment }}"
tasks:
- block:

View File

@@ -2,6 +2,7 @@
- hosts: 127.0.0.1
connection: local
gather_facts: no
environment: "{{ ansible_test.environment }}"
tasks:
- block:

View File

@@ -2,6 +2,7 @@
- hosts: 127.0.0.1
connection: local
gather_facts: no
environment: "{{ ansible_test.environment }}"
tasks:
- block:

View File

@@ -2,11 +2,12 @@
- hosts: 127.0.0.1
connection: local
gather_facts: no
environment: "{{ ansible_test.environment }}"
tasks:
- block:
- set_fact:
instance_id: 'rds-mariadb-{{ resource_prefix }}'
instance_id: '{{ resource_prefix }}-mariadb'
- name: assert group was populated with inventory but is empty
assert:
@@ -16,36 +17,28 @@
# Create new host, add it to inventory and then terminate it without updating the cache
# TODO: Uncomment once rds_instance has been added
#- name: set connection information for all tasks
# set_fact:
# aws_connection_info: &aws_connection_info
# aws_access_key: '{{ aws_access_key }}'
# aws_secret_key: '{{ aws_secret_key }}'
# security_token: '{{ security_token }}'
# region: '{{ aws_region }}'
# no_log: yes
- name: set connection information for all tasks
set_fact:
aws_connection_info: &aws_connection_info
aws_access_key: '{{ aws_access_key }}'
aws_secret_key: '{{ aws_secret_key }}'
security_token: '{{ security_token }}'
region: '{{ aws_region }}'
no_log: yes
- name: Use AWS CLI to create an RDS DB instance
command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'"
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
# TODO: Uncomment once rds_instance has been added
#- name: create minimal mariadb instance in default VPC and default subnet group
# rds_instance:
# state: present
# engine: mariadb
# instance_class: db.t2.micro
# storage: 20
# instance_id: 'rds-mariadb-{{ resource_prefix }}'
# master_username: 'ansible-test-user'
# master_password: 'password-{{ resource_prefix }}'
# <<: *aws_connection_info
# register: setup_instance
- name: create minimal mariadb instance in default VPC and default subnet group
rds_instance:
state: present
engine: mariadb
db_instance_class: db.t2.micro
allocated_storage: 20
instance_id: '{{ instance_id }}'
master_username: 'ansibletestuser'
master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
tags:
workload_type: other
<<: *aws_connection_info
register: setup_instance
- meta: refresh_inventory
@@ -55,22 +48,12 @@
always:
- name: Use AWS CLI to delete the DB instance
command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
ignore_errors: True
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
# TODO: Uncomment once rds_instance has been added
#- name: remove mariadb instance
# rds_instance:
# state: absent
# engine: mariadb
# skip_final_snapshot: yes
# instance_id: ansible-rds-mariadb-example
# <<: *aws_connection_info
# ignore_errors: yes
# when: setup_instance is defined
- name: remove mariadb instance
rds_instance:
state: absent
engine: mariadb
skip_final_snapshot: yes
instance_id: '{{ instance_id }}'
<<: *aws_connection_info
ignore_errors: yes
when: setup_instance is defined

View File

@@ -2,6 +2,7 @@
- hosts: 127.0.0.1
connection: local
gather_facts: no
environment: "{{ ansible_test.environment }}"
tasks:
- block:
@@ -18,35 +19,28 @@
# Create new host, refresh inventory, remove host, refresh inventory
#- name: set connection information for all tasks
# set_fact:
# aws_connection_info: &aws_connection_info
# aws_access_key: '{{ aws_access_key }}'
# aws_secret_key: '{{ aws_secret_key }}'
# security_token: '{{ security_token }}'
# region: '{{ aws_region }}'
# no_log: yes
- name: set connection information for all tasks
set_fact:
aws_connection_info: &aws_connection_info
aws_access_key: '{{ aws_access_key }}'
aws_secret_key: '{{ aws_secret_key }}'
security_token: '{{ security_token }}'
region: '{{ aws_region }}'
no_log: yes
- name: Use AWS CLI to create an RDS DB instance
command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'"
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
# TODO: Uncomment once rds_instance has been added
#- name: create minimal mariadb instance in default VPC and default subnet group
# rds_instance:
# state: present
# engine: mariadb
# instance_class: db.t2.micro
# storage: 20
# instance_id: 'rds-mariadb-{{ resource_prefix }}'
# master_username: 'ansible-test-user'
# master_password: 'password-{{ resource_prefix }}'
# <<: *aws_connection_info
# register: setup_instance
- name: create minimal mariadb instance in default VPC and default subnet group
rds_instance:
state: present
engine: mariadb
db_instance_class: db.t2.micro
allocated_storage: 20
instance_id: '{{ instance_id }}'
master_username: 'ansibletestuser'
master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
tags:
workload_type: other
<<: *aws_connection_info
register: setup_instance
- meta: refresh_inventory
@@ -57,23 +51,13 @@
- "groups.aws_rds | length == 1"
- "groups.aws_rds.0 == '{{ instance_id }}'"
- name: Use AWS CLI to delete the DB instance
command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
ignore_errors: True
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
# TODO: Uncomment once rds_instance has been added
#- name: remove mariadb instance
# rds_instance:
# state: absent
# engine: mariadb
# skip_final_snapshot: yes
# instance_id: ansible-rds-mariadb-example
# <<: *aws_connection_info
- name: remove mariadb instance
rds_instance:
state: absent
engine: mariadb
skip_final_snapshot: yes
instance_id: '{{ instance_id }}'
<<: *aws_connection_info
- meta: refresh_inventory
@@ -85,22 +69,12 @@
always:
- name: Use AWS CLI to delete the DB instance
command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
ignore_errors: True
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
# TODO: Uncomment once rds_instance has been added
#- name: remove mariadb instance
# rds_instance:
# state: absent
# engine: mariadb
# skip_final_snapshot: yes
# instance_id: ansible-rds-mariadb-example
# <<: *aws_connection_info
# ignore_errors: yes
# when: setup_instance is defined
- name: remove mariadb instance
rds_instance:
state: absent
engine: mariadb
skip_final_snapshot: yes
instance_id: '{{ instance_id }}'
<<: *aws_connection_info
ignore_errors: yes
when: setup_instance is defined

View File

@@ -2,44 +2,38 @@
- hosts: 127.0.0.1
connection: local
gather_facts: no
environment: "{{ ansible_test.environment }}"
tasks:
- block:
- set_fact:
instance_id: "{{ resource_prefix }}constructed"
instance_id: "{{ resource_prefix }}-mariadb"
# Create new host, refresh inventory
#- name: set connection information for all tasks
# set_fact:
# aws_connection_info: &aws_connection_info
# aws_access_key: '{{ aws_access_key }}'
# aws_secret_key: '{{ aws_secret_key }}'
# security_token: '{{ security_token }}'
# region: '{{ aws_region }}'
# no_log: yes
- name: set connection information for all tasks
set_fact:
aws_connection_info: &aws_connection_info
aws_access_key: '{{ aws_access_key }}'
aws_secret_key: '{{ aws_secret_key }}'
security_token: '{{ security_token }}'
region: '{{ aws_region }}'
no_log: yes
# TODO: Uncomment once rds_instance has been added
#- name: create minimal mariadb instance in default VPC and default subnet group
# rds_instance:
# state: present
# engine: mariadb
# instance_class: db.t2.micro
# storage: 20
# instance_id: 'rds-mariadb-{{ resource_prefix }}'
# master_username: 'ansible-test-user'
# master_password: 'password-{{ resource_prefix }}'
# <<: *aws_connection_info
# register: setup_instance
- name: Use AWS CLI to create an RDS DB instance
command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser' --tags Key='workload_type',Value='other'"
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
- name: create minimal mariadb instance in default VPC and default subnet group
rds_instance:
state: present
engine: mariadb
db_instance_class: db.t2.micro
allocated_storage: 20
instance_id: '{{ resource_prefix }}-mariadb'
master_username: 'ansibletestuser'
master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
tags:
workload_type: other
<<: *aws_connection_info
register: setup_instance
- meta: refresh_inventory
- debug: var=groups
@@ -51,26 +45,16 @@
- "groups | length == 6"
- groups.tag_workload_type_other
- groups.rds_mariadb
- groups.rds_parameter_group_default_mariadb10_0
- groups.rds_parameter_group_default_mariadb10_3
always:
- name: Use AWS CLI to delete the DB instance
command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
ignore_errors: True
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
# TODO: Uncomment once rds_instance has been added
#- name: remove mariadb instance
# rds_instance:
# state: absent
# engine: mariadb
# skip_final_snapshot: yes
# instance_id: ansible-rds-mariadb-example
# <<: *aws_connection_info
# ignore_errors: yes
# when: setup_instance is defined
- name: remove mariadb instance
rds_instance:
state: absent
engine: mariadb
skip_final_snapshot: yes
instance_id: '{{ instance_id }}'
<<: *aws_connection_info
ignore_errors: yes
when: setup_instance is defined

View File

@@ -9,34 +9,28 @@
- "'aws_rds' in groups"
- "not groups.aws_rds"
#- name: set connection information for all tasks
# set_fact:
# aws_connection_info: &aws_connection_info
# aws_access_key: "{{ aws_access_key }}"
# aws_secret_key: "{{ aws_secret_key }}"
# security_token: "{{ security_token }}"
# region: "{{ aws_region }}"
# no_log: yes
- name: set connection information for all tasks
set_fact:
aws_connection_info: &aws_connection_info
aws_access_key: "{{ aws_access_key }}"
aws_secret_key: "{{ aws_secret_key }}"
security_token: "{{ security_token }}"
region: "{{ aws_region }}"
no_log: yes
#- name: create minimal mariadb instance in default VPC and default subnet group
# rds_instance:
# state: present
# engine: mariadb
# instance_class: db.t2.micro
# storage: 20
# instance_id: 'rds-mariadb-{{ resource_prefix }}'
# master_username: 'ansible-test-user'
# master_password: 'password-{{ resource_prefix }}'
# <<: *aws_connection_info
# register: setup_instance
- name: Use AWS CLI to create an RDS DB instance
command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'"
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
- name: create minimal mariadb instance in default VPC and default subnet group
rds_instance:
state: present
engine: mariadb
db_instance_class: db.t2.micro
allocated_storage: 20
instance_id: 'rds-mariadb-{{ resource_prefix }}'
master_username: 'ansibletestuser'
master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
tags:
workload_type: other
<<: *aws_connection_info
register: setup_instance
- meta: refresh_inventory
@@ -47,22 +41,13 @@
- "groups.aws_rds | length == 1"
- "groups.aws_rds.0 == '{{ resource_prefix }}'"
#- name: remove mariadb instance
# rds_instance:
# state: absent
# engine: mariadb
# skip_final_snapshot: yes
# instance_id: ansible-rds-mariadb-example
# <<: *aws_connection_info
- name: Use AWS CLI to delete the DB instance
command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
ignore_errors: True
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
- name: remove mariadb instance
rds_instance:
state: absent
engine: mariadb
skip_final_snapshot: yes
instance_id: ansible-rds-mariadb-example
<<: *aws_connection_info
- meta: refresh_inventory
@@ -74,21 +59,12 @@
always:
- name: Use AWS CLI to delete the DB instance
command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot"
ignore_errors: True
environment:
AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
AWS_SESSION_TOKEN: "{{ security_token }}"
AWS_DEFAULT_REGION: "{{ aws_region }}"
#- name: remove mariadb instance
# rds_instance:
# state: absent
# engine: mariadb
# skip_final_snapshot: yes
# instance_id: ansible-rds-mariadb-example
# <<: *aws_connection_info
# ignore_errors: yes
# when: setup_instance is defined
- name: remove mariadb instance
rds_instance:
state: absent
engine: mariadb
skip_final_snapshot: yes
instance_id: ansible-rds-mariadb-example
<<: *aws_connection_info
ignore_errors: yes
when: setup_instance is defined

View File

@@ -4,3 +4,5 @@ aws_secret_access_key: '{{ aws_secret_key }}'
aws_security_token: '{{ security_token }}'
regions:
- '{{ aws_region }}'
filters:
db-instance-id: "{{ resource_prefix }}-mariadb"

View File

@@ -7,3 +7,5 @@ aws_secret_access_key: '{{ aws_secret_key }}'
aws_security_token: '{{ security_token }}'
regions:
- '{{ aws_region }}'
filters:
db-instance-id: "{{ resource_prefix }}-mariadb"

View File

@@ -11,3 +11,5 @@ keyed_groups:
prefix: tag
- key: engine
prefix: rds
filters:
db-instance-id: "{{ resource_prefix }}-mariadb"