Python yaml.safe_load_all() Examples

def get_template_contexts(file_path):
    try:
        with open(file_path) as f:
            try:
                contexts = yaml.safe_load_all(f.read())
            except Exception as e:
                raise RuntimeError('Unable to load yaml file: {}, {}'.format(file_path, e))

            for context in contexts:
                if context is None:
                    continue  # Skip empty YAML documents
                if 'kind' not in context or context['kind'] is None:
                    raise RuntimeError('Field "kind" not found (or empty) in file "{}"'.format(file_path))
                if 'metadata' not in context or context['metadata'] is None:
                    raise RuntimeError('Field "metadata" not found (or empty) in file "{}"'.format(file_path))
                if 'name' not in context['metadata'] or context['metadata']['name'] is None:
                    raise RuntimeError('Field "metadata->name" not found (or empty) in file "{}"'.format(file_path))
                if 'spec' in context:
                    # INFO: Set replicas = 1 by default for replaces cases in Deployment and StatefulSet
                    if 'replicas' not in context['spec'] or context['spec']['replicas'] is None:
                        if context['kind'] in ['Deployment', 'StatefulSet']:
                            context['spec']['replicas'] = 1
                yield context
    except FileNotFoundError as e:
        raise RuntimeError(e) 

def test_update_chart_group_document_valid(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))
        documents_modified = copy.deepcopy(documents)
        documents_modified[1]['data']['sequenced'] = True

        # starting out, both doc have different values for data
        self.assertNotEqual(documents[1], documents_modified[1])

        ovr = Override(documents)
        # update with document values with the modified ones
        ovr.update_document(documents_modified[1])

        # after the update, both documents are equal
        self.assertEqual(
            ovr.documents[1]['data']['sequenced'],
            documents_modified[1]['data']['sequenced'])
        self.assertEqual(ovr.documents[1], documents_modified[1]) 

def test_summary_user_data(self):
        user_data1 = {'a': 1}
        user_data2 = {'b': 1}
        user_data = [user_data1, user_data2]
        dump_path = os.path.join(self.tmp_path, 'ha.yaml')
        writer = records.TestSummaryWriter(dump_path)
        for data in user_data:
            writer.dump(data, records.TestSummaryEntryType.USER_DATA)
        with io.open(dump_path, 'r', encoding='utf-8') as f:
            contents = []
            for c in yaml.safe_load_all(f):
                contents.append(c)
        for content in contents:
            self.assertEqual(content['Type'],
                             records.TestSummaryEntryType.USER_DATA.value)
        self.assertEqual(contents[0]['a'], user_data1['a'])
        self.assertEqual(contents[1]['b'], user_data2['b']) 

def test_update_armada_manifest_valid(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))
        documents_modified = copy.deepcopy(documents)
        documents_modified[2]['data']['release_prefix'] = 'armada-modified'

        # starting out, both doc have different values for data
        self.assertNotEqual(documents[2], documents_modified[2])

        ovr = Override(documents)
        # update with document values with the modified ones
        ovr.update_armada_manifest(documents_modified[2])

        # after the update, both documents are equal
        self.assertEqual(ovr.documents[2]['data']['release_prefix'],
                         documents_modified[2]['data']['release_prefix'])
        self.assertEqual(ovr.documents[2], documents_modified[2]) 

def test_update_chart_group_document_keys_not_removed_with_override(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))

        documents_modified = copy.deepcopy(documents)
        del documents_modified[1]['data']['sequenced']

        # verify both doc have different values for data
        self.assertNotEqual(documents[1], documents_modified[1])

        ovr = Override(documents)
        # update with document values with the modified ones
        ovr.update_chart_group_document(documents_modified[1])

        self.assertIn('sequenced', ovr.documents[1]['data'])
        self.assertNotEqual(ovr.documents[1], documents_modified[1]) 

def test_update_armada_manifest_keys_not_removed_with_override(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))

        documents_modified = copy.deepcopy(documents)
        del documents_modified[2]['data']['release_prefix']

        # verify both doc have different values for data
        self.assertNotEqual(documents[2], documents_modified[2])

        ovr = Override(documents)
        # update with document values from base_manifest
        ovr.update_armada_manifest(documents_modified[2])

        self.assertIn('release_prefix', ovr.documents[2]['data'])
        self.assertNotEqual(ovr.documents[2], documents_modified[2]) 

def test_update_dictionary_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '01')
        merge = "{}/templates/override-{}.yaml".format(self.basepath, '01')

        with open(self.base_manifest) as f, open(expected) as e, open(
                merge) as m:
            merging_values = list(yaml.safe_load_all(m.read()))
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['chart', 'blog-1']
            ovr = Override(documents)
            ovr.update_document(merging_values)
            ovr_doc = ovr.find_manifest_document(doc_path)
            expect_doc = list(yaml.load_all(e.read()))[0]

            self.assertEqual(ovr_doc, expect_doc) 

def test_write_user_data(self):
        content = {'a': 1}

        class MockBaseTest(base_test.BaseTestClass):
            def test_something(self):
                self.record_data(content)

        bt_cls = MockBaseTest(self.mock_test_cls_configs)
        bt_cls.run(test_names=["test_something"])
        actual_record = bt_cls.results.passed[0]
        self.assertEqual(actual_record.test_name, "test_something")
        hit = False
        with io.open(self.summary_file, 'r', encoding='utf-8') as f:
            for c in yaml.safe_load_all(f):
                if c['Type'] != records.TestSummaryEntryType.USER_DATA.value:
                    continue
                hit = True
                self.assertEqual(c['a'], content['a'])
                self.assertIsNotNone(c['timestamp'])
        self.assertTrue(hit) 

def _loadRelease(self):
  release = self.dict.get('release', {})
  if release.get('type') == 'github' and 'user' in release and 'repo' in release:
   for dict in GithubApi(release['user'], release['repo'], self.repo.client).getReleases():
    asset = self._findGithubAsset(dict.get('assets', []))
    if asset:
     return {
      'version': dict.get('name') or dict.get('tag_name'),
      'date': datetime.strptime(dict.get('created_at'), '%Y-%m-%dT%H:%M:%SZ'),
      'desc': dict.get('body'),
      'url': asset,
     }
  elif release.get('type') == 'yaml' and 'url' in release:
   for dict in yaml.safe_load_all(http.get(release['url']).data):
    if 'version' in dict and 'url' in dict:
     return dict
  elif 'version' in release and 'url' in release:
   return release 

def test_pre_flight_ops(self, mock_tiller, mock_source):
        """Test pre-flight checks and operations."""
        yaml_documents = list(yaml.safe_load_all(TEST_YAML))
        armada_obj = armada.Armada(yaml_documents)

        # Mock methods called by `pre_flight_ops()`.
        mock_tiller.tiller_status.return_value = True
        mock_source.git_clone.return_value = CHART_SOURCES[0][0]

        self._test_pre_flight_ops(armada_obj)

        mock_tiller.assert_called_once_with(tiller_host=None,
                                            tiller_namespace='kube-system',
                                            tiller_port=44134)
        mock_source.git_clone.assert_called_once_with(
            'git://github.com/dummy/armada', 'master', auth_method=None,
            proxy_server=None) 

def load_schemas(self):
        self.v1_doc_schemas = dict()
        schema_dir = self._get_schema_dir()

        for schema_file in os.listdir(schema_dir):
            f = open(os.path.join(schema_dir, schema_file), 'r')
            for schema in yaml.safe_load_all(f):
                schema_for = schema['metadata']['name']
                if schema_for in self.v1_doc_schemas:
                    self.logger.warning(
                        "Duplicate document schemas found for document kind %s."
                        % schema_for)
                self.logger.debug(
                    "Loaded schema for document kind %s." % schema_for)
                self.v1_doc_schemas[schema_for] = schema
            f.close() 

def test_set_list_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '03')

        with open(self.base_manifest) as f, open(expected) as e:
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['manifest', 'simple-armada']
            override = ('manifest:simple-armada:chart_groups=\
                         blog-group3,blog-group4',)
            ovr = Override(documents, override)
            ovr.update_manifests()
            ovr_doc = ovr.find_manifest_document(doc_path)
            target_docs = list(yaml.load_all(e.read()))
            expected_doc = [x
                            for x
                            in target_docs
                            if x.get('schema') == 'armada/Manifest/v1'][0]
            self.assertEqual(expected_doc.get('data'), ovr_doc.get('data')) 

def load_schemas(self):
        self.v1_doc_schemas = dict()
        schema_dir = self._get_schema_dir()

        for schema_file in os.listdir(schema_dir):
            f = open(os.path.join(schema_dir, schema_file), 'r')
            for schema in yaml.safe_load_all(f):
                schema_for = schema['metadata']['name']
                if schema_for in self.v1_doc_schemas:
                    self.logger.warning(
                        "Duplicate document schemas found for document kind %s."
                        % schema_for)
                self.logger.debug(
                    "Loaded schema for document kind %s." % schema_for)
                self.v1_doc_schemas[schema_for] = schema.get('data')
            f.close() 

def req_yaml(self, req, default=None):
        if req.content_length is None or req.content_length == 0:
            return default

        raw_body = req.stream.read(req.content_length or 0)

        if raw_body is None:
            return default

        try:
            return list(yaml.safe_load_all(raw_body.decode('utf-8')))
        except yaml.YAMLError:
            with excutils.save_and_reraise_exception():
                self.error(
                    req.context,
                    "Invalid YAML in request: \n%s" % raw_body.decode('utf-8')) 

def req_yaml(self, req, default=None):
        if req.content_length is None or req.content_length == 0:
            return default

        raw_body = req.stream.read(req.content_length or 0)

        if raw_body is None:
            return default

        try:
            return list(yaml.safe_load_all(raw_body.decode('utf-8')))
        except yaml.YAMLError:
            with excutils.save_and_reraise_exception():
                self.error(
                    req.context,
                    "Invalid YAML in request: \n%s" % raw_body.decode('utf-8')) 

def _load_schemas():
    """Populates ``_SCHEMAS`` with the schemas defined in package
    ``armada.schemas``.

    """
    schema_dir = _get_schema_dir()
    for schema_file in os.listdir(schema_dir):
        with open(os.path.join(schema_dir, schema_file)) as f:
            for schema in yaml.safe_load_all(f):
                name = schema['metadata']['name']
                if name in _SCHEMAS:
                    raise RuntimeError(
                        'Duplicate schema specified for: %s.' % name)
                _SCHEMAS[name] = _get_schema_info(name, schema['data'])


# Fill the cache. 

def test_update_dictionary_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '01')
        merge = "{}/templates/override-{}.yaml".format(self.basepath, '01')

        with open(self.base_manifest) as f, open(expected) as e, open(
                merge) as m:
            merging_values = list(yaml.safe_load_all(m.read()))
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['chart', 'blog-1']
            ovr = Override(documents)
            ovr.update_documents(merging_values)
            ovr_doc = ovr.find_manifest_document(doc_path)
            expect_doc = list(yaml.load_all(e.read()))[0]

            self.assertEqual(ovr_doc, expect_doc) 

def test_update_armada_manifest_keys_not_removed_with_override(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))

        documents_modified = copy.deepcopy(documents)
        del documents_modified[2]['data']['release_prefix']

        # verify both doc have different values for data
        self.assertNotEqual(documents[2], documents_modified[2])

        ovr = Override(documents)
        # update with document values from base_manifest
        ovr.update_document(documents_modified[2])

        self.assertIn('release_prefix', ovr.documents[2]['data'])
        self.assertNotEqual(ovr.documents[2], documents_modified[2]) 

def test_own_document_examples(self):
        examples_path = os.path.join(os.getcwd(), 'examples')
        example_files = [
            os.path.join(examples_path, f) for f in os.listdir(examples_path)
            if os.path.isfile(os.path.join(examples_path, f))
        ]
        validated_manifests = []

        for example_file in example_files:
            with open(example_file) as f:
                documents = yaml.safe_load_all(f.read())

            # If the example file doesn't have a document with
            # armada/Manifest/v1 then skip validating it as the example could
            # merely be an override.
            has_manifest = any(
                x['schema'] == 'armada/Manifest/v1' for x in documents)
            if not has_manifest:
                continue

            validated_manifests.append(example_file)
            valid, _ = validate.validate_armada_documents(list(documents))
            self.assertTrue(valid)

        self.assertTrue(validated_manifests) 

def test_update_armada_manifest_valid(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))
        documents_modified = copy.deepcopy(documents)
        documents_modified[2]['data']['release_prefix'] = 'armada-modified'

        # starting out, both doc have different values for data
        self.assertNotEqual(documents[2], documents_modified[2])

        ovr = Override(documents)
        # update with document values with the modified ones
        ovr.update_document(documents_modified[2])

        # after the update, both documents are equal
        self.assertEqual(
            ovr.documents[2]['data']['release_prefix'],
            documents_modified[2]['data']['release_prefix'])
        self.assertEqual(ovr.documents[2], documents_modified[2]) 

def test_validate_invalid_chart_armada_manifest(self):
        template = '{}/resources/valid_armada_document.yaml'.format(
            self.basepath)

        with open(template) as f:
            documents = list(yaml.safe_load_all(f.read()))

        mariadb_document = [
            d for d in documents if d['metadata']['name'] == 'mariadb'
        ][0]
        del mariadb_document['data']['release']

        _, error_messages = validate.validate_armada_documents(documents)
        expected_error = self._build_error_message(
            'armada/Chart/v1', 'mariadb', "'release' is a required property")

        self.assertEqual(1, len(error_messages))
        self.assertEqual(expected_error, error_messages[0]['message']) 

def test_validate_invalid_chart_armada_manifest(self):
        template = '{}/resources/valid_armada_document.yaml'.format(
            self.basepath)

        with open(template) as f:
            documents = list(yaml.safe_load_all(f.read()))

        mariadb_document = [
            d for d in documents if d['metadata']['name'] == 'mariadb'][0]
        del mariadb_document['data']['release']

        _, error_messages = validate.validate_armada_documents(documents)
        expected_error = self._build_error_message(
            'armada/Chart/v1', 'mariadb',
            "'release' is a required property")

        self.assertEqual(1, len(error_messages))
        self.assertEqual(expected_error, error_messages[0]['message']) 

def test_update_chart_group_document_keys_not_removed_with_override(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))

        documents_modified = copy.deepcopy(documents)
        del documents_modified[1]['data']['sequenced']

        # verify both doc have different values for data
        self.assertNotEqual(documents[1], documents_modified[1])

        ovr = Override(documents)
        # update with document values with the modified ones
        ovr.update_document(documents_modified[1])

        self.assertIn('sequenced', ovr.documents[1]['data'])
        self.assertNotEqual(ovr.documents[1], documents_modified[1]) 

def test_own_document_examples(self):
        examples_path = os.path.join(os.getcwd(), 'examples')
        example_files = [
            os.path.join(examples_path, f) for f in os.listdir(examples_path)
            if os.path.isfile(os.path.join(examples_path, f))
        ]
        validated_manifests = []

        for example_file in example_files:
            with open(example_file) as f:
                documents = yaml.safe_load_all(f.read())

            # If the example file doesn't have a document with
            # armada/Manifest/v1 then skip validating it as the example could
            # merely be an override.
            has_manifest = any(
                x['schema'] == 'armada/Manifest/v1' for x in documents)
            if not has_manifest:
                continue

            validated_manifests.append(example_file)
            valid, _ = validate.validate_armada_documents(list(documents))
            self.assertTrue(valid)

        self.assertTrue(validated_manifests) 

def test_test_controller_with_manifest(self, mock_tiller, mock_manifest):
        rules = {'armada:tests_manifest': '@'}
        self.policy.set_rules(rules)

        manifest_path = os.path.join(os.getcwd(), 'examples',
                                     'keystone-manifest.yaml')
        with open(manifest_path, 'r') as f:
            payload = f.read()
        documents = list(yaml.safe_load_all(payload))

        resp = self.app.simulate_post('/api/v1.0/tests', body=payload)
        self.assertEqual(200, resp.status_code)

        result = json.loads(resp.text)
        expected = {
            "tests": {"passed": [], "skipped": [], "failed": []}
        }
        self.assertEqual(expected, result)

        mock_manifest.assert_called_once_with(
            documents, target_manifest=None)
        self.assertTrue(mock_tiller.called) 

def test_update_chart_document_keys_not_removed_with_override(self):
        with open(self.base_manifest) as f:
            documents = list(yaml.safe_load_all(f.read()))

        documents_modified = copy.deepcopy(documents)
        del documents_modified[0]['data']['chart_name']

        # verify both doc have different values for data
        self.assertNotEqual(documents[0], documents_modified[0])

        ovr = Override(documents)
        # update with document values with the modified ones
        ovr.update_chart_document(documents_modified[0])

        self.assertIn('chart_name', ovr.documents[0]['data'])
        self.assertNotEqual(ovr.documents[0], documents_modified[0]) 

def test_armada_get_manifest_exception(self, MockChartDownload):
        """Test armada handling with invalid manifest."""
        yaml_documents = list(yaml.safe_load_all(TEST_YAML))
        error_re = ('.*Documents must include at least one of each of .*')
        self.assertRaisesRegexp(
            ManifestException, error_re, armada.Armada, yaml_documents[:1],
            mock.MagicMock()) 

def _get_chart_by_name(self, name):
        name = name.split('armada-')[-1]
        yaml_documents = list(yaml.safe_load_all(TEST_YAML))
        return [
            c for c in yaml_documents if c['data'].get('chart_name') == name
        ][0] 

def test_update_manifests_invalid_override(self):
        with open(self.base_manifest) as f:
            original_documents = list(yaml.safe_load_all(f.read()))

        override = ('manifest:simple-armada:name=' 'overridden', )
        ovr = Override(original_documents, override)
        self.assertRaises(
            override_exceptions.InvalidOverrideValueException,
            ovr.update_manifests) 

def invoke(self):
        with Tiller(tiller_host=self.tiller_host, tiller_port=self.tiller_port,
                    tiller_namespace=self.tiller_namespace,
                    bearer_token=self.bearer_token) as tiller:

            try:
                doc_data = ReferenceResolver.resolve_reference(
                    self.location, k8s=tiller.k8s)
                documents = list()
                for d in doc_data:
                    documents.extend(list(yaml.safe_load_all(d.decode())))
            except InvalidPathException as ex:
                self.logger.error(str(ex))
                return
            except yaml.YAMLError as yex:
                self.logger.error("Invalid YAML found: %s" % str(yex))
                return

            try:
                resp = self.handle(documents, tiller)
                self.output(resp)
            finally:
                if self.metrics_output:
                    path = self.metrics_output
                    self.logger.info(
                        'Storing metrics output in path: {}'.format(path))
                    prometheus_client.write_to_textfile(path, metrics.REGISTRY)