EFS - add support for new Provisioned Throughput (#43253)

* efs.py: Add support for EFS provisioned throughput

* efs_facts.py: Add support for EFS provisioned throughput

* efs_facts integration tests updated with provision throughput

* efs_facts: Tests refactoring - add failure and success playbook according to botocore version.

* efs_facts: More tests and new option descriptions adjustment

* efs_facts tests renamed to efs

test/integration/targets/efs/aliases

@@ -1,2 +1,3 @@
 cloud/aws
 unsupported
+efs_facts

test/integration/targets/efs/playbooks/full_test.yml

@@ -0,0 +1,8 @@
+- hosts: localhost
+  connection: local
+
+  vars:
+    resource_prefix: 'ansible-testing'
+
+  roles:
+    - efs

test/integration/targets/efs/playbooks/roles/efs/tasks/main.yml

@@ -66,6 +66,7 @@
         targets:
             - subnet_id: "{{testing_subnet_a.subnet.id}}"
             - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'bursting'
       register: created_efs
 
     # ============================================================
@@ -99,6 +100,7 @@
           - efs_result.ansible_facts.efs[0].encrypted == false
           - efs_result.ansible_facts.efs[0].life_cycle_state == "available"
           - efs_result.ansible_facts.efs[0].performance_mode == "generalPurpose"
+          - efs_result.ansible_facts.efs[0].throughput_mode  == "bursting"
           - efs_result.ansible_facts.efs[0].mount_targets[0].security_groups[0] == vpc_default_sg_id
           - efs_result.ansible_facts.efs[0].mount_targets[1].security_groups[0] == vpc_default_sg_id
 
@@ -161,6 +163,90 @@
     - assert:
         that: "{{efs_result_assertions}}"
 
+    # ============================================================
+    #   Not checking efs_result.efs["throughput_mode"] here as
+    #   Efs with status "life_cycle_state": "updating" might return the previous values
+    - name: Update Efs to use provisioned throughput_mode
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'provisioned'
+        provisioned_throughput_in_mibps: 5.0
+      register: efs_result
+
+    - assert: 
+        that: 
+          - efs_result is changed
+
+    # ============================================================
+    - name: Efs same value for provisioned_throughput_in_mibps
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'provisioned'
+        provisioned_throughput_in_mibps: 5.0
+      register: efs_result
+ 
+    - assert:
+        that: 
+          - efs_result is not changed
+          - efs_result.efs["throughput_mode"]  == "provisioned"
+          - efs_result.efs["provisioned_throughput_in_mibps"] == 5.0
+
+    # ============================================================
+    - name: Efs new value for provisioned_throughput_in_mibps
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'provisioned'
+        provisioned_throughput_in_mibps: 8.0
+      register: efs_result
+
+    - assert:
+        that:
+          - efs_result is changed
+          - efs_result.efs["provisioned_throughput_in_mibps"] == 8.0
+
+    # ============================================================
+    - name: Check new facts with provisioned mode
+      efs_facts:
+        name: "{{ resource_prefix }}-test-efs"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - set_fact:
+        efs_result_assertions:
+          - efs_result is not changed
+          - efs_result.ansible_facts.efs[0].throughput_mode  == "provisioned"
+          - efs_result.ansible_facts.efs[0].provisioned_throughput_in_mibps == 8.0
+          - (efs_result.ansible_facts.efs | length) == 1
+          - efs_result.ansible_facts.efs[0].creation_token == "{{ resource_prefix }}-test-efs"
+          - efs_result.ansible_facts.efs[0].file_system_id == created_efs.efs.file_system_id
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
     # ============================================================
     - name: Query unknown EFS by tag
       efs_facts:

test/integration/targets/efs/playbooks/version_fail.yml

@@ -0,0 +1,31 @@
+- hosts: localhost
+  connection: local
+  vars:
+     resource_prefix: 'ansible-testing'
+
+  tasks:
+    - block:
+        - name: set up aws connection info
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: "{{ aws_access_key }}"
+              aws_secret_key: "{{ aws_secret_key }}"
+              security_token: "{{ security_token }}"
+              region: "{{ aws_region }}"
+          no_log: True
+
+        - name: create efs with provisioned_throughput options (fails gracefully)
+          efs:
+            state: present
+            name: "{{ resource_prefix }}-efs"
+            throughput_mode: 'provisioned'
+            provisioned_throughput_in_mibps: 8.0
+            <<: *aws_connection_info
+          register: efs_provisioned_throughput_creation
+          ignore_errors: yes
+
+        - name: check that graceful error message is returned when creation with throughput_mode and old botocore
+          assert:
+            that:
+              - efs_provisioned_throughput_creation.failed
+              - 'efs_provisioned_throughput_creation.msg == "throughput_mode parameter requires botocore >= 1.10.57"'

test/integration/targets/efs/runme.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+# This is needed for the ubuntu1604py3 tests
+# Ubuntu patches virtualenv to make the default python2
+# but for the python3 tests we need virtualenv to use python3
+PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
+# Test graceful failure for older versions of botocore
+export ANSIBLE_ROLES_PATH=../
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-less-than-1.10.57"
+source "${MYTMPDIR}/botocore-less-than-1.10.57/bin/activate"
+"${PYTHON}" -m pip install 'botocore<1.10.57' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/version_fail.yml "$@"
+# Run full test suite
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent"
+source "${MYTMPDIR}/botocore-recent/bin/activate"
+$PYTHON -m pip install 'botocore>=1.10.57' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"