Implement vault encrypted yaml variables. (#16274)

Make !vault-encrypted create a AnsibleVaultUnicode
yaml object that can be used as a regular string object.

This allows a playbook to include a encrypted vault
blob for the value of a yaml variable. A 'secret_password'
variable can have it's value encrypted instead of having
to vault encrypt an entire vars file.

Add __ENCRYPTED__ to the vault yaml types so
template.Template can treat it similar
to __UNSAFE__ flags.

vault.VaultLib api changes:
    - Split VaultLib.encrypt to encrypt and encrypt_bytestring

    - VaultLib.encrypt() previously accepted the plaintext data
      as either a byte string or a unicode string.
      Doing the right thing based on the input type would fail
      on py3 if given a arg of type 'bytes'. To simplify the
      API, vaultlib.encrypt() now assumes input plaintext is a
      py2 unicode or py3 str. It will encode to utf-8 then call
      the new encrypt_bytestring(). The new methods are less
      ambiguous.

    - moved VaultLib.is_encrypted logic to vault module scope
      and split to is_encrypted() and is_encrypted_file().

Add a test/unit/mock/yaml_helper.py
It has some helpers for testing parsing/yaml

Integration tests added as roles test_vault and test_vault_embedded

test/integration/Makefile

@@ -162,6 +162,8 @@ test_vault: setup
 	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --list-hosts -e outputdir=$(TEST_DIR)  -e @$(VARS_FILE)
 	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --syntax-check -e outputdir=$(TEST_DIR)  -e @$(VARS_FILE)
 	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
+	ansible-playbook test_vault_embedded.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --syntax-check -e outputdir=$(TEST_DIR)  -e @$(VARS_FILE)
+	ansible-playbook test_vault_embedded.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
 
 # test_delegate_to does not work unless we have permission to ssh to localhost.
 # Would take some more effort on our test systems to implement that -- probably

test/integration/roles/test_vault/tasks/main.yml

@@ -0,0 +1,7 @@
+
+- assert:
+    that:
+      - 'secret_var == "secret"'
+
+
+- copy: src=vault-secret.txt dest={{output_dir}}/secret.txt

test/integration/roles/test_vault/vars/main.yml

@@ -0,0 +1,9 @@
+$ANSIBLE_VAULT;1.1;AES256
+31626536666232643662346539623662393436386162643439643434656231343435653936343235
+6139346364396166336636383734333430373763336434310a303137623539653939336132626234
+64613232396532313731313935333433353330666466646663303233323331636234326464643166
+6538653264636166370a613161313064653566323037393962643032353230396536313865326362
+34396262303130326632623162623230346238633932393938393766313036643835613936356233
+33323730373331386337353339613165373064323134343930333031623036326164353534646631
+31313963666234623731316238656233396638643331306231373539643039383434373035306233
+30386230363730643561

test/integration/roles/test_vault_embedded/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+- name: Assert that a embedded vault of a string with no newline works
+  assert:
+    that:
+      - '"{{ vault_encrypted_one_line_var }}" == "Setec Astronomy"'
+
+- name: Assert that a multi line embedded vault works, including new line
+  assert:
+    that:
+      - vault_encrypted_var == "Setec Astronomy\n"
+
+# TODO: add a expected fail here
+# - debug: var=vault_encrypted_one_line_var_with_embedded_template
+

test/integration/roles/test_vault_embedded/vars/main.yml

@@ -0,0 +1,17 @@
+# If you use normal 'ansible-vault create' or edit, files always have at least one new line
+# so c&p from a vault encrypted that wasn't specifically created sans new line ends up with one.
+# (specifically created, as in 'echo -n "just one line" > my_secret.yml'
+vault_encrypted_var: !vault-encrypted |
+              $ANSIBLE_VAULT;1.1;AES256
+              66386439653236336462626566653063336164663966303231363934653561363964363833313662
+              6431626536303530376336343832656537303632313433360a626438346336353331386135323734
+              62656361653630373231613662633962316233633936396165386439616533353965373339616234
+              3430613539666330390a313736323265656432366236633330313963326365653937323833366536
+              34623731376664623134383463316265643436343438623266623965636363326136
+vault_encrypted_one_line_var: !vault-encrypted |
+                              $ANSIBLE_VAULT;1.1;AES256
+                              33363965326261303234626463623963633531343539616138316433353830356566396130353436
+                              3562643163366231316662386565383735653432386435610a306664636137376132643732393835
+                              63383038383730306639353234326630666539346233376330303938323639306661313032396437
+                              6233623062366136310a633866373936313238333730653739323461656662303864663666653563
+                              3138

test/integration/test_vault.yml

@@ -1,13 +1,4 @@
 - hosts: testhost
-  vars_files:
-    - vars/test_var_encrypted.yml
-  
   gather_facts: False
-
-  tasks:
-    - assert:
-        that:
-          - 'secret_var == "secret"'
-          
-    - copy: src=vault-secret.txt dest={{output_dir}}/secret.txt
-
+  roles:
+    - { role: test_vault, tags: test_vault}

test/integration/test_vault_embedded.yml

@@ -0,0 +1,4 @@
+- hosts: testhost
+  gather_facts: False
+  roles:
+    - { role: test_vault_embedded, tags: test_vault_embedded}