parse botocore.endpoint logs into a list of AWS actions (#49312)

* Add an option to parse botocore.endpoint logs for the AWS actions performed during a task

Add a callback to consolidate all AWS actions used by modules

Added some documentation to the AWS guidelines

* Enable aws_resource_actions callback only for AWS tests

* Add script to help generate policies

* Set debug_botocore_endpoint_logs via environment variable for all AWS integration tests

Ensure AWS tests inherit environment

(also remove AWS CLI in aws_rds inventory tests and use the module)

lib/ansible/module_utils/aws/core.py

@@ -60,10 +60,18 @@ don't need to be wrapped in the backoff decorator.
 
 """
 
+import re
+import logging
 import traceback
 from functools import wraps
 from distutils.version import LooseVersion
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    # Python 3
+    from io import StringIO
+
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils._text import to_native
 from ansible.module_utils.ec2 import HAS_BOTO3, camel_dict_to_snake_dict, ec2_argument_spec, boto3_conn, get_aws_connection_info
@@ -120,14 +128,38 @@ class AnsibleAWSModule(object):
         self._diff = self._module._diff
         self._name = self._module._name
 
+        self._botocore_endpoint_log_stream = StringIO()
+        self.logger = None
+        if self.params.get('debug_botocore_endpoint_logs'):
+            self.logger = logging.getLogger('botocore.endpoint')
+            self.logger.setLevel(logging.DEBUG)
+            self.logger.addHandler(logging.StreamHandler(self._botocore_endpoint_log_stream))
+
     @property
     def params(self):
         return self._module.params
 
+    def _get_resource_action_list(self):
+        actions = []
+        for ln in self._botocore_endpoint_log_stream.getvalue().split('\n'):
+            ln = ln.strip()
+            if not ln:
+                continue
+            found_operational_request = re.search(r"OperationModel\(name=.*?\)", ln)
+            if found_operational_request:
+                operation_request = found_operational_request.group(0)[20:-1]
+                resource = re.search(r"https://.*?\.", ln).group(0)[8:-1]
+                actions.append("{0}:{1}".format(resource, operation_request))
+        return list(set(actions))
+
     def exit_json(self, *args, **kwargs):
+        if self.params.get('debug_botocore_endpoint_logs'):
+            kwargs['resource_actions'] = self._get_resource_action_list()
         return self._module.exit_json(*args, **kwargs)
 
     def fail_json(self, *args, **kwargs):
+        if self.params.get('debug_botocore_endpoint_logs'):
+            kwargs['resource_actions'] = self._get_resource_action_list()
         return self._module.fail_json(*args, **kwargs)
 
     def debug(self, *args, **kwargs):
@@ -190,7 +222,7 @@ class AnsibleAWSModule(object):
         if response is not None:
             failure.update(**camel_dict_to_snake_dict(response))
 
-        self._module.fail_json(**failure)
+        self.fail_json(**failure)
 
     def _gather_versions(self):
         """Gather AWS SDK (boto3 and botocore) dependency versions

lib/ansible/module_utils/ec2.py

@@ -31,7 +31,7 @@ import re
 import traceback
 
 from ansible.module_utils.ansible_release import __version__
-from ansible.module_utils.basic import missing_required_lib
+from ansible.module_utils.basic import missing_required_lib, env_fallback
 from ansible.module_utils._text import to_native, to_text
 from ansible.module_utils.cloud import CloudRetry
 from ansible.module_utils.six import string_types, binary_type, text_type
@@ -177,6 +177,7 @@ def boto_exception(err):
 
 def aws_common_argument_spec():
     return dict(
+        debug_botocore_endpoint_logs=dict(fallback=(env_fallback, ['ANSIBLE_DEBUG_BOTOCORE_LOGS']), default=False, type='bool'),
         ec2_url=dict(),
         aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True),
         aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),