Implement vault encrypted yaml variables. (#16274)

Make !vault-encrypted create a AnsibleVaultUnicode
yaml object that can be used as a regular string object.

This allows a playbook to include a encrypted vault
blob for the value of a yaml variable. A 'secret_password'
variable can have it's value encrypted instead of having
to vault encrypt an entire vars file.

Add __ENCRYPTED__ to the vault yaml types so
template.Template can treat it similar
to __UNSAFE__ flags.

vault.VaultLib api changes:
    - Split VaultLib.encrypt to encrypt and encrypt_bytestring

    - VaultLib.encrypt() previously accepted the plaintext data
      as either a byte string or a unicode string.
      Doing the right thing based on the input type would fail
      on py3 if given a arg of type 'bytes'. To simplify the
      API, vaultlib.encrypt() now assumes input plaintext is a
      py2 unicode or py3 str. It will encode to utf-8 then call
      the new encrypt_bytestring(). The new methods are less
      ambiguous.

    - moved VaultLib.is_encrypted logic to vault module scope
      and split to is_encrypted() and is_encrypted_file().

Add a test/unit/mock/yaml_helper.py
It has some helpers for testing parsing/yaml

Integration tests added as roles test_vault and test_vault_embedded

test/units/parsing/yaml/test_dumper.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import io
+import yaml
+
+try:
+    from _yaml import ParserError
+except ImportError:
+    from yaml.parser import ParserError
+
+from ansible.parsing.yaml import dumper
+from ansible.parsing.yaml.loader import AnsibleLoader
+
+from ansible.compat.tests import unittest
+from ansible.parsing.yaml import objects
+from ansible.parsing import vault
+
+from units.mock.yaml_helper import YamlTestUtils
+
+class TestAnsibleDumper(unittest.TestCase, YamlTestUtils):
+    def setUp(self):
+        self.vault_password = "hunter42"
+        self.good_vault = vault.VaultLib(self.vault_password)
+        self.vault = self.good_vault
+        self.stream = self._build_stream()
+        self.dumper = dumper.AnsibleDumper
+
+    def _build_stream(self,yaml_text=None):
+        text = yaml_text or u''
+        stream = io.StringIO(text)
+        return stream
+
+    def _loader(self, stream):
+        return AnsibleLoader(stream, vault_password=self.vault_password)
+
+    def test(self):
+        plaintext = 'This is a string we are going to encrypt.'
+        avu = objects.AnsibleVaultEncryptedUnicode.from_plaintext(plaintext, vault=self.vault)
+
+        yaml_out = self._dump_string(avu, dumper=self.dumper)
+        stream = self._build_stream(yaml_out)
+        loader = self._loader(stream)
+
+        data_from_yaml = loader.get_single_data()
+
+        self.assertEquals(plaintext, data_from_yaml.data)

test/units/parsing/yaml/test_loader.py

@@ -26,20 +26,26 @@ from six import text_type, binary_type
 from collections import Sequence, Set, Mapping
 
 from ansible.compat.tests import unittest
-from ansible.compat.tests.mock import patch
 
+from ansible import errors
 from ansible.parsing.yaml.loader import AnsibleLoader
+from ansible.parsing import vault
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.parsing.yaml.dumper import AnsibleDumper
+from ansible.utils.unicode import to_bytes
+
+from units.mock.yaml_helper import YamlTestUtils
 
 try:
     from _yaml import ParserError
 except ImportError:
     from yaml.parser import ParserError
 
-
 class NameStringIO(StringIO):
     """In py2.6, StringIO doesn't let you set name because a baseclass has it
     as readonly property"""
     name = None
+
     def __init__(self, *args, **kwargs):
         super(NameStringIO, self).__init__(*args, **kwargs)
 
@@ -159,6 +165,124 @@ class TestAnsibleLoaderBasic(unittest.TestCase):
         self.assertEqual(data[0][u'baz'].ansible_pos, ('myfile.yml', 2, 9))
 
 
+class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
+    def setUp(self):
+        self.vault_password = "hunter42"
+        self.vault = vault.VaultLib(self.vault_password)
+
+    def test_wrong_password(self):
+        plaintext = u"Ansible"
+        bob_password = "this is a different password"
+
+        bobs_vault = vault.VaultLib(bob_password)
+
+        ciphertext = bobs_vault.encrypt(plaintext)
+
+        try:
+            self.vault.decrypt(ciphertext)
+        except Exception as e:
+            self.assertIsInstance(e, errors.AnsibleError)
+            self.assertEqual(e.message, 'Decryption failed')
+
+    def _encrypt_plaintext(self, plaintext):
+        # Construct a yaml repr of a vault by hand
+        vaulted_var_bytes = self.vault.encrypt(plaintext)
+
+        # add yaml tag
+        vaulted_var = vaulted_var_bytes.decode()
+        lines = vaulted_var.splitlines()
+        lines2 = []
+        for line in lines:
+            lines2.append('        %s' % line)
+
+        vaulted_var = '\n'.join(lines2)
+        tagged_vaulted_var = u"""!vault-encrypted |\n%s""" % vaulted_var
+        return tagged_vaulted_var
+
+    def _build_stream(self, yaml_text):
+        stream = NameStringIO(yaml_text)
+        stream.name = 'my.yml'
+        return stream
+
+    def _loader(self, stream):
+        return AnsibleLoader(stream, vault_password=self.vault_password)
+
+    def _load_yaml(self, yaml_text, password):
+        stream = self._build_stream(yaml_text)
+        loader = self._loader(stream)
+
+        data_from_yaml = loader.get_single_data()
+
+        return data_from_yaml
+
+    def test_dump_load_cycle(self):
+        avu = AnsibleVaultEncryptedUnicode.from_plaintext('The plaintext for test_dump_load_cycle.', vault=self.vault)
+        self._dump_load_cycle(avu)
+
+    def test_embedded_vault_from_dump(self):
+        avu = AnsibleVaultEncryptedUnicode.from_plaintext('setec astronomy', vault=self.vault)
+        blip = {'stuff1': [{'a dict key': 24},
+                           {'shhh-ssh-secrets': avu,
+                            'nothing to see here': 'move along'}],
+                'another key': 24.1}
+
+        blip = ['some string', 'another string', avu]
+        stream = NameStringIO()
+
+        self._dump_stream(blip, stream, dumper=AnsibleDumper)
+
+        print(stream.getvalue())
+        stream.seek(0)
+
+        stream.seek(0)
+
+        loader = self._loader(stream)
+
+        data_from_yaml = loader.get_data()
+        stream2 = NameStringIO(u'')
+        # verify we can dump the object again
+        self._dump_stream(data_from_yaml, stream2, dumper=AnsibleDumper)
+
+    def test_embedded_vault(self):
+        plaintext_var = u"""This is the plaintext string."""
+        tagged_vaulted_var = self._encrypt_plaintext(plaintext_var)
+        another_vaulted_var = self._encrypt_plaintext(plaintext_var)
+
+        different_var = u"""A different string that is not the same as the first one."""
+        different_vaulted_var = self._encrypt_plaintext(different_var)
+
+        yaml_text = u"""---\nwebster: daniel\noed: oxford\nthe_secret: %s\nanother_secret: %s\ndifferent_secret: %s""" % (tagged_vaulted_var, another_vaulted_var, different_vaulted_var)
+
+        data_from_yaml = self._load_yaml(yaml_text, self.vault_password)
+        vault_string = data_from_yaml['the_secret']
+
+        self.assertEquals(plaintext_var, data_from_yaml['the_secret'])
+
+        test_dict = {}
+        test_dict[vault_string] = 'did this work?'
+
+        self.assertEquals(vault_string.data, vault_string)
+
+        # This looks weird and useless, but the object in question has a custom __eq__
+        self.assertEquals(vault_string, vault_string)
+
+        another_vault_string = data_from_yaml['another_secret']
+        different_vault_string = data_from_yaml['different_secret']
+
+        self.assertEquals(vault_string, another_vault_string)
+        self.assertNotEquals(vault_string, different_vault_string)
+
+        # More testing of __eq__/__ne__
+        self.assertTrue('some string' != vault_string)
+        self.assertNotEquals('some string', vault_string)
+
+        # Note this is a compare of the str/unicode of these, they are diferent types
+        # so we want to test self == other, and other == self etc
+        self.assertEquals(plaintext_var, vault_string)
+        self.assertEquals(vault_string, plaintext_var)
+        self.assertFalse(plaintext_var != vault_string)
+        self.assertFalse(vault_string != plaintext_var)
+
 class TestAnsibleLoaderPlay(unittest.TestCase):
 
     def setUp(self):
@@ -242,7 +366,7 @@ class TestAnsibleLoaderPlay(unittest.TestCase):
 
     def check_vars(self):
         # Numbers don't have line/col information yet
-        #self.assertEqual(self.data[0][u'vars'][u'number'].ansible_pos, (self.play_filename, 4, 21))
+        # self.assertEqual(self.data[0][u'vars'][u'number'].ansible_pos, (self.play_filename, 4, 21))
 
         self.assertEqual(self.data[0][u'vars'][u'string'].ansible_pos, (self.play_filename, 5, 29))
         self.assertEqual(self.data[0][u'vars'][u'utf8_string'].ansible_pos, (self.play_filename, 6, 34))
@@ -255,8 +379,8 @@ class TestAnsibleLoaderPlay(unittest.TestCase):
         self.assertEqual(self.data[0][u'vars'][u'list'][0].ansible_pos, (self.play_filename, 11, 25))
         self.assertEqual(self.data[0][u'vars'][u'list'][1].ansible_pos, (self.play_filename, 12, 25))
         # Numbers don't have line/col info yet
-        #self.assertEqual(self.data[0][u'vars'][u'list'][2].ansible_pos, (self.play_filename, 13, 25))
-        #self.assertEqual(self.data[0][u'vars'][u'list'][3].ansible_pos, (self.play_filename, 14, 25))
+        # self.assertEqual(self.data[0][u'vars'][u'list'][2].ansible_pos, (self.play_filename, 13, 25))
+        # self.assertEqual(self.data[0][u'vars'][u'list'][3].ansible_pos, (self.play_filename, 14, 25))
 
     def check_tasks(self):
         #

test/units/parsing/yaml/test_objects.py

@@ -0,0 +1,129 @@
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Copyright 2016, Adrian Likins <alikins@redhat.com>
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.compat.tests import unittest
+
+
+from ansible.parsing import vault
+from ansible.parsing.yaml.loader import AnsibleLoader
+
+# module under test
+from ansible.parsing.yaml import objects
+
+from units.mock.yaml_helper import YamlTestUtils
+
+
+class TestAnsibleVaultUnicodeNoVault(unittest.TestCase, YamlTestUtils):
+    def test_empty_init(self):
+        self.assertRaises(TypeError, objects.AnsibleVaultEncryptedUnicode)
+
+    def test_empty_string_init(self):
+        seq = ''.encode('utf8')
+        self.assert_values(seq)
+
+    def test_empty_byte_string_init(self):
+        seq = b''
+        self.assert_values(seq)
+
+    def _assert_values(self, avu, seq):
+        self.assertIsInstance(avu, objects.AnsibleVaultEncryptedUnicode)
+        self.assertTrue(avu.vault is None)
+        # AnsibleVaultEncryptedUnicode without a vault should never == any string
+        self.assertNotEquals(avu, seq)
+
+    def assert_values(self, seq):
+        avu = objects.AnsibleVaultEncryptedUnicode(seq)
+        self._assert_values(avu, seq)
+
+    def test_single_char(self):
+        seq = 'a'.encode('utf8')
+        self.assert_values(seq)
+
+    def test_string(self):
+        seq = 'some letters'
+        self.assert_values(seq)
+
+    def test_byte_string(self):
+        seq = 'some letters'.encode('utf8')
+        self.assert_values(seq)
+
+
+class TestAnsibleVaultEncryptedUnicode(unittest.TestCase, YamlTestUtils):
+    def setUp(self):
+        self.vault_password = "hunter42"
+        self.good_vault = vault.VaultLib(self.vault_password)
+
+        self.wrong_vault_password = 'not-hunter42'
+        self.wrong_vault = vault.VaultLib(self.wrong_vault_password)
+
+        self.vault = self.good_vault
+
+    def _loader(self, stream):
+        return AnsibleLoader(stream, vault_password=self.vault_password)
+
+    def test_dump_load_cycle(self):
+        aveu = self._from_plaintext('the test string for TestAnsibleVaultEncryptedUnicode.test_dump_load_cycle')
+        self._dump_load_cycle(aveu)
+
+    def assert_values(self, avu, seq):
+        self.assertIsInstance(avu, objects.AnsibleVaultEncryptedUnicode)
+
+        self.assertEquals(avu, seq)
+        self.assertTrue(avu.vault is self.vault)
+        self.assertIsInstance(avu.vault, vault.VaultLib)
+
+    def _from_plaintext(self, seq):
+        return objects.AnsibleVaultEncryptedUnicode.from_plaintext(seq, vault=self.vault)
+
+    def _from_ciphertext(self, ciphertext):
+        avu = objects.AnsibleVaultEncryptedUnicode(ciphertext)
+        avu.vault = self.vault
+        return avu
+
+    def test_empty_init(self):
+        self.assertRaises(TypeError, objects.AnsibleVaultEncryptedUnicode)
+
+    def test_empty_string_init_from_plaintext(self):
+        seq = ''
+        avu = self._from_plaintext(seq)
+        self.assert_values(avu,seq)
+
+    def test_empty_unicode_init_from_plaintext(self):
+        seq = u''
+        avu = self._from_plaintext(seq)
+        self.assert_values(avu,seq)
+
+    def test_string_from_plaintext(self):
+        seq = 'some letters'
+        avu = self._from_plaintext(seq)
+        self.assert_values(avu,seq)
+
+    def test_unicode_from_plaintext(self):
+        seq = u'some letters'
+        avu = self._from_plaintext(seq)
+        self.assert_values(avu,seq)
+
+    # TODO/FIXME: make sure bad password fails differently than 'thats not encrypted'
+    def test_empty_string_wrong_password(self):
+        seq = ''
+        self.vault = self.wrong_vault
+        avu = self._from_plaintext(seq)
+        self.assert_values(avu, seq)