fix decrypted vault utf8 values (#37539)

* Fix errors decrypted non-ascii vault vars

AnsibleVaultEncryptedUnicode was just using b"".decode()
instead of to_text() on the bytestrings returned from
vault.decrypt() and could cause errors on python2
if non-ascii since decode() defaults to ascii.
Use to_text() to default to decoding utf-8.

add intg and unit tests for value of vaulted vars
being non-ascii utf8

based on https://github.com/ansible/ansible/issues/37258

Fixes #37258

* yamllint fixups

test/integration/targets/vault/host_vars/myhost.yml

@@ -0,0 +1,7 @@
+myvar: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          31356335363836383937363933366135623233343830326234633633623734336636343630396464
+          3234343638313166663237343536646336323862613739380a346266316336356230643838663031
+          34623034383639323062373235356564393337346666393665313237313231306131356637346537
+          3966393238666430310a363462326639323033653237373036643936613234623063643761663033
+          3832

test/integration/targets/vault/host_vars/testhost.yml

@@ -0,0 +1,7 @@
+vaulted_utf8_value: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          39313961356631343234656136636231663539363963386364653436346133366366633031366364
+          3332376636333837333036633662316135383365343335380a393331663434663238666537343163
+          62363561336431623666633735313766613663333736653064373632666131356434336537383336
+          3333343436613232330a643461363831633166333237653530353131316361643465353132616362
+          3461

test/integration/targets/vault/runme.sh

@@ -386,7 +386,7 @@ echo "rc was $WRONG_RC (5 is expected)"
 
 # encrypt with a password from a vault encrypted password file and multiple vault-ids
 # but this time specify with --encrypt-vault-id, but specifying vault-id names (instead of default)
-# ansible-vault encrypt "$@" --vault-id from_vault_password@vault-password --vault-id from_encrypted_vault_password@encrypted-vault-password --encrypt-vault-id from_encrypted_vault_password "${TEST_FILE_ENC_PASSWORD}"
+# ansible-vault encrypt "$@" --vault-id from_vault_password@vault-password --vault-id from_encrypted_vault_password@encrypted-vault-password --encrypt-vault-id from_encrypted_vault_password "${TEST_FILE(_ENC_PASSWORD}"
 
 # try to view the file encrypted with the vault-password we didnt specify
 # to verify we didnt choose the wrong vault-id
@@ -418,6 +418,9 @@ ansible-playbook test_vault_embedded.yml -i ../../inventory -v "$@" --vault-pass
 ansible-playbook test_vaulted_inventory.yml -i vaulted.inventory -v "$@" --vault-password-file vault-password
 ansible-playbook test_vaulted_template.yml -i ../../inventory -v "$@" --vault-password-file vault-password
 
+# test a playbook with a host_var whose value is non-ascii utf8 (see https://github.com/ansible/ansible/issues/37258)
+ansible-playbook -i ../../inventory -v "$@" --vault-id vault-password test_vaulted_utf8_value.yml
+
 # test with password from password script
 ansible-playbook test_vault.yml          -i ../../inventory -v "$@" --vault-password-file password-script.py
 ansible-playbook test_vault_embedded.yml -i ../../inventory -v "$@" --vault-password-file password-script.py

test/integration/targets/vault/test_vaulted_utf8_value.yml

@@ -0,0 +1,15 @@
+- name: "test that the vaulted_utf8_value decrypts correctly"
+  gather_facts: false
+  hosts: testhost
+  vars:
+    expected: "aöffü"
+  tasks:
+    - name: decrypt vaulted_utf8_value and show it in debug
+      debug:
+         var: vaulted_utf8_value
+
+    - name: assert decrypted vaulted_utf8_value matches expected
+      assert:
+        that:
+          - "vaulted_utf8_value == expected"
+          - "vaulted_utf8_value == 'aöffü'"