Python yaml.dump_all() Examples

def save_current_file():
	print("Saving current file: {}".format(current_file))
	def order_dict(d):
		return OrderedDict([
			("title", d["title"]),
			("type", d["type"]),
			("has_source", d["has_source"]),
			("info", OrderedDict([
				(key, d["info"][key] if key in d["info"] else "") for key in info_keys
			])),
			("streams", OrderedDict([
				(key, d["streams"][key] if key in d["streams"] else "") for key in stream_keys
			]))
		])
	
	try:
		sorted_docs = [order_dict(doc) for doc in current_docs]
		with open(current_file, "w", encoding="UTF-8") as f:
			yaml.dump_all(sorted_docs, f, default_flow_style=False, indent=4, allow_unicode=True)
	except:
		from traceback import print_exc
		print_exc()
		return False
	return True 

def main():
  parser = ArgumentParser()
  parser.add_argument(
      "-m",
      "--manifest",
      dest="manifest",
      help="the manifest file to be parsed and updated")
  parser.add_argument(
      "-a",
      "--appname",
      dest="application_name",
      help="the application instance name")
  args = parser.parse_args()
  manifest = args.manifest
  app_name = args.application_name
  resources = load_resources_yaml(manifest)
  resources = [ensure_resource_has_app_label(r, app_name) for r in resources]
  with open(manifest, "w", encoding='utf-8') as out:
    yaml.dump_all(resources, out, default_flow_style=False, explicit_start=True) 

def save_current_file():
	print("Saving current file: {}".format(current_file))
	def order_dict(d):
		return OrderedDict([
			("title", d["title"]),
			("type", d["type"]),
			("has_source", d["has_source"]),
			("info", OrderedDict([
				(key, d["info"][key] if key in d["info"] else "") for key in info_keys
			])),
			("streams", OrderedDict([
				(key, d["streams"][key] if key in d["streams"] else "") for key in stream_keys
			]))
		])
	
	try:
		sorted_docs = [order_dict(doc) for doc in current_docs]
		with open(current_file, "w", encoding="UTF-8") as f:
			yaml.dump_all(sorted_docs, f, default_flow_style=False, indent=4, allow_unicode=True)
	except:
		from traceback import print_exc
		print_exc()
		return False
	return True 

def monkey_patch_pyyaml_reverse():
    global pyyaml_load
    global pyyaml_load_all
    global pyyaml_dump_all

    if pyyaml_load:
        log.info("reversing monkey patch for yaml.load...")
        yaml.load = pyyaml_load
        pyyaml_load = None
    if pyyaml_load_all:
        log.info("reversing monkey patch for yaml.load_all...")
        yaml.load_all = pyyaml_load_all
        pyyaml_load_all = None
    if pyyaml_dump_all:
        log.info("reversing monkey patch for yaml.dump_all... (affects all yaml dump operations)")
        yaml.dump_all = pyyaml_dump_all
        pyyaml_dump_all = None 

def monkey_patch_pyyaml():
    global pyyaml_load
    global pyyaml_load_all
    global pyyaml_dump_all

    if not pyyaml_load:
        log.info("monkey patching yaml.load...")
        pyyaml_load = yaml.load
        yaml.load = safe_yaml_load
    if not pyyaml_load_all:
        log.info("monkey patching yaml.load_all...")
        pyyaml_load_all = yaml.load_all
        yaml.load_all = safe_yaml_load_all
    if not pyyaml_dump_all:
        log.info("monkey patching yaml.dump_all... (affects all yaml dump operations)")
        pyyaml_dump_all = yaml.dump_all
        yaml.dump_all = safe_yaml_dump_all 

def test_create_configdocs_409(*args):
    err_resp = stubs.gen_err_resp(message='Invalid collection',
                                  sub_message='Buffer is either not...',
                                  sub_error_count=1,
                                  sub_info_count=0,
                                  reason='Buffermode : append',
                                  code=409)
    responses.add(responses.POST,
                  'http://shiptest/configdocs/design',
                  body=err_resp,
                  status=409)

    filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
    document_data = yaml.dump_all(filename)
    file_list = (filename,)

    response = CreateConfigdocs(stubs.StubCliContext(),
                                'design',
                                'append',
                                False,
                                document_data,
                                file_list).invoke_and_return_resp()
    assert 'Error: Invalid collection' in response
    assert 'Reason: Buffermode : append' in response
    assert 'Buffer is either not...' in response 

def test_create_configdocs(*args):
    succ_resp = stubs.gen_err_resp(message='Validations succeeded',
                                   sub_error_count=0,
                                   sub_info_count=0,
                                   reason='Validation',
                                   code=200)
    responses.add(responses.POST,
                  'http://shiptest/configdocs/design',
                  body=succ_resp,
                  status=201)

    filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
    document_data = yaml.dump_all(filename)
    file_list = (filename,)

    response = CreateConfigdocs(stubs.StubCliContext(),
                                'design',
                                'append',
                                False,
                                document_data,
                                file_list).invoke_and_return_resp()
    assert 'Configuration documents added.'
    assert 'Status: Validations succeeded' in response
    assert 'Reason: Validation' in response 

def formatted_response_handler(response):
    """Base format handler for either json or yaml depending on call"""
    call = response.headers['Content-Type']
    if 'json' in call:
        try:
            return json.dumps(response.json(), sort_keys=True, indent=4)
        except ValueError:
            return (
                "This is not json and could not be printed as such. \n" +
                response.text
            )

    else:  # all others should be yaml
        try:
            return (yaml.dump_all(
                yaml.safe_load_all(response.content),
                width=79,
                indent=4,
                default_flow_style=False))
        except ValueError:
            return (
                "This is not yaml and could not be printed as such.\n" +
                response.text
            ) 

def __init__(self, name, obj_dict_list, project, tmp_path):
        """
        Args:
            name (str): Name of this object config file
            obj_dict_list (list): List of dictionaries with k8s objects
            project (ocp.OCP): Instance of :class:`ocp.OCP` of ``Project``
                kind, specifying namespace where the object will be deployed.
            tmp_path (pathlib.Path): Directory where a temporary yaml file will
                be created. In test context, use pytest fixture `tmp_path`_.

        .. _`tmp_path`: https://docs.pytest.org/en/latest/tmpdir.html#the-tmp-path-fixture
        """
        self.name = name
        self.project = project
        # dump the job description in yaml format into a temporary file
        self._tmp_path = tmp_path
        self.yaml_file = tmp_path / f"objectconfig.{self.name}.yaml"
        self.yaml_file.write_text(yaml.dump_all(obj_dict_list)) 

def dump_data_to_temp_yaml(data, temp_yaml):
    """
    Dump data to temporary yaml file

    Args:
        data (dict or list): dict or list (in case of multi_document) with
            data to dump to the yaml file.
        temp_yaml (str): file path of yaml file

    Returns:
        str: dumped yaml data

    """
    dumper = yaml.dump if isinstance(data, dict) else yaml.dump_all
    yaml_data = dumper(data)
    with open(temp_yaml, 'w') as yaml_file:
        yaml_file.write(yaml_data)
    if isinstance(data, dict):
        yaml_data_censored = dumper(censor_values(deepcopy(data)))
    else:
        yaml_data_censored = [
            dumper(censor_values(deepcopy(doc))) for doc in data
        ]
    logger.info(yaml_data_censored)
    return yaml_data 

def test_create_configdocs_201_with_val_fails(*args):
    succ_resp = stubs.gen_err_resp(message='Validations failed',
                                   sub_message='Some reason',
                                   sub_error_count=2,
                                   sub_info_count=1,
                                   reason='Validation',
                                   code=400)
    responses.add(responses.POST,
                  'http://shiptest/configdocs/design',
                  body=succ_resp,
                  status=201)

    filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
    document_data = yaml.dump_all(filename)
    file_list = (filename,)

    response = CreateConfigdocs(stubs.StubCliContext(),
                                'design',
                                'append',
                                False,
                                document_data,
                                file_list).invoke_and_return_resp()
    assert 'Configuration documents added.' in response
    assert 'Status: Validations failed' in response
    assert 'Reason: Validation' in response
    assert 'Some reason-1' in response 

def dump_yaml(self, file_path=None):
        """
        :param file_path: If self.file_path is None, it must be passed
        :type file_path:
        :return:
        :rtype:
        """
        file_path = file_path or self.file_path
        assert_or_raise(file_path is not None, UnityDocumentError("file_path parameter must be passed"))
        with open(file_path, 'w', newline=self.newline) as fp:
            yaml.dump_all(self.data, stream=fp, Dumper=UnityDumper) 

def test_load_multiple_fails(self):
        example = [{"a": "b"}, {"c": "d"}]

        with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as wrapped_tmp:
            # put into a file
            dumped = yaml.dump_all(example)
            wrapped_tmp.write(dumped.encode("utf8"))
            wrapped_tmp.close()

            try:
                with pytest.raises(exceptions.UnexpectedDocumentsError):
                    load_single_document_yaml(wrapped_tmp.name)
            finally:
                os.remove(wrapped_tmp.name) 

def create_season_config(config, db, output_file):
	info("Checking for new shows")
	shows = _get_primary_source_shows(config)
	
	debug("Outputting new shows")
	with open(output_file, "w", encoding="utf-8") as f:
		yaml.dump_all(shows, f, explicit_start=True, default_flow_style=False) 

def test_create_configdocs_empty(*args):
    def validating_callback(request):
        # a request that has empty_collection should have no body.
        assert request.body is None
        resp_body = stubs.gen_err_resp(
            message='Validations succeeded',
            sub_error_count=0,
            sub_info_count=0,
            reason='Validation',
            code=200)
        return (201, {}, resp_body)

    responses.add_callback(
        responses.POST,
        'http://shiptest/configdocs/design',
        callback=validating_callback,
        content_type='application/json')

    filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
    document_data = yaml.dump_all(filename)
    file_list = (filename, )

    # pass data and empty_collection = True - should init with data, but
    # not send the data on invoke
    action = CreateConfigdocs(
        stubs.StubCliContext(),
        collection='design',
        buffer_mode='append',
        empty_collection=True,
        data=document_data,
        filenames=file_list)

    assert action.data == document_data
    assert action.empty_collection == True

    response = action.invoke_and_return_resp()
    assert response.startswith("Configuration documents added.") 

def test_monkey_patch():
    assert yaml.dump_all == safe_yaml_dump_all
    assert yaml.load_all == safe_yaml_load_all
    assert yaml.load == safe_yaml_load 

def test_unsafe():
    dummy = Dummy()

    with pytest.raises(yaml.representer.RepresenterError):
        yaml.dump_all([dummy])

    with pytest.raises(yaml.representer.RepresenterError):
        yaml.dump(dummy, Dumper=yDumper)

    # reverse monkey patch and try again
    monkey_patch_pyyaml_reverse()

    with tempfile.TemporaryFile(suffix='.yaml', mode='w+t') as f:
        yaml.dump_all([dummy], stream=f)
        f.seek(0)  # rewind

        doc_unsafe = yaml.load(f)
        assert type(doc_unsafe) is Dummy

        monkey_patch_pyyaml()
        with pytest.raises(yaml.constructor.ConstructorError):
            f.seek(0)  # rewind
            safe_yaml_load(f)

        with pytest.raises(yaml.constructor.ConstructorError):
            f.seek(0)  # rewind
            yaml.load(f) 

def safe_dump(data, **kwargs):
    kwargs["default_flow_style"] = False
    return yaml.dump_all(
        [data], None, Dumper=PrettyPrinterDumper, **kwargs) 

def main():
  parser = ArgumentParser()
  parser.add_argument(
      "--manifest", help="the manifest file location to be cleared of tests")
  parser.add_argument(
      "--deploy_tests",
      action="store_true",
      help="indicates whether tests should be deployed")
  args = parser.parse_args()

  manifest = args.manifest
  resources = load_resources_yaml(manifest)
  filtered_resources = []
  for resource in resources:
    helm_hook = deep_get(resource, "metadata", "annotations", _HELM_HOOK_KEY)
    if helm_hook is None:
      filtered_resources.append(resource)
    elif helm_hook == _HOOK_SUCCESS:
      if args.deploy_tests:
        annotations = deep_get(resource, "metadata", "annotations")
        del annotations[_HELM_HOOK_KEY]
        annotations[GOOGLE_CLOUD_TEST] = "test"
        filtered_resources.append(resource)
    elif helm_hook == _HOOK_FAILURE:
      if args.deploy_tests:
        raise Exception("Helm hook {} is not supported".format(helm_hook))
    else:
      raise Exception("Helm hook {} is not supported".format(helm_hook))

  with open(manifest, "w", encoding='utf-8') as out:
    yaml.dump_all(
        filtered_resources, out, default_flow_style=False, explicit_start=True) 

def save_info(path, info, default_flow_style=False):
    with open(path, "w") as file:
        _yaml.dump_all(info, file, default_flow_style=default_flow_style) 

def dump(item):
    return dump_all([item]) 

def dump_all(items):
    return yaml.dump_all(
        list(map(_reorder, items)),
        None,
        _build_dumper(),
        default_flow_style=False) 

def _write(self, output_dir):
        documents = self.get_documents()
        with open(os.path.join(output_dir, 'certificates.yaml'), 'w') as f:
            # Don't use safe_dump_all so we can block format certificate data.
            yaml.dump_all(
                documents,
                stream=f,
                default_flow_style=False,
                explicit_start=True,
                indent=2) 

def services(generation_counter, kind):
    if kind in app.elements:
        return yaml.dump_all(app.elements[kind], Dumper=yaml_dumper), 200
    else:
        return "no such element", 404 

def dump(value: SequenceView):
    st = dump_all(value, default_flow_style=False)
    if not st.startswith('---'):
        st = '---\n' + st
    return st 

def cluster(kind_cluster) -> Generator[dict, None, None]:
    docker_image = os.getenv("TEST_IMAGE")
    kind_cluster.load_docker_image(docker_image)

    logging.info("Deploying kube-web-view ...")
    deployment_manifests_path = Path(__file__).parent / "deployment.yaml"

    kubectl = kind_cluster.kubectl

    with NamedTemporaryFile(mode="w+") as tmp:
        with deployment_manifests_path.open() as f:
            resources = list(yaml.safe_load_all(f))
        dep = resources[-1]
        assert (
            dep["kind"] == "Deployment" and dep["metadata"]["name"] == "kube-web-view"
        )
        dep["spec"]["template"]["spec"]["containers"][0]["image"] = docker_image
        yaml.dump_all(documents=resources, stream=tmp)
        kubectl("apply", "-f", tmp.name)

    logging.info("Deploying other test resources ...")
    kubectl("apply", "-f", str(Path(__file__).parent / "test-resources.yaml"))

    logging.info("Waiting for rollout ...")
    kubectl("rollout", "status", "deployment/kube-web-view")

    with kind_cluster.port_forward("service/kube-web-view", 80) as port:
        url = f"http://localhost:{port}/"
        yield {"url": url} 

def create_season_config(config, db, output_file):
	info("Checking for new shows")
	shows = _get_primary_source_shows(config)
	
	debug("Outputting new shows")
	with open(output_file, "w", encoding="utf-8") as f:
		yaml.dump_all(shows, f, explicit_start=True, default_flow_style=False) 

def test_bad_yaml_usage(self):
        python_node = self.get_ast_node(
            """
            import yaml

            var1 = {'foo': 'bar'}
            var2 = 'test: !!python/object/apply:print ["HAI"]'

            yaml.dump(var1)
            yaml.dump_all([var1])

            yaml.load(var2)
            yaml.load_all(var2)
            """
        )

        linter = dlint.linters.BadYAMLUseLinter()
        linter.visit(python_node)

        result = linter.get_results()
        expected = [
            dlint.linters.base.Flake8Result(
                lineno=7,
                col_offset=0,
                message=dlint.linters.BadYAMLUseLinter._error_tmpl
            ),
            dlint.linters.base.Flake8Result(
                lineno=8,
                col_offset=0,
                message=dlint.linters.BadYAMLUseLinter._error_tmpl
            ),
            dlint.linters.base.Flake8Result(
                lineno=10,
                col_offset=0,
                message=dlint.linters.BadYAMLUseLinter._error_tmpl
            ),
            dlint.linters.base.Flake8Result(
                lineno=11,
                col_offset=0,
                message=dlint.linters.BadYAMLUseLinter._error_tmpl
            ),
        ]

        assert result == expected 

def brozzler_list_sites(argv=None):
    argv = argv or sys.argv
    arg_parser = argparse.ArgumentParser(
            prog=os.path.basename(argv[0]),
            formatter_class=BetterArgumentDefaultsHelpFormatter)
    arg_parser.add_argument(
            '--yaml', dest='yaml', action='store_true', help=(
                'yaml output (default is json)'))
    group = arg_parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
            '--active', dest='active', action='store_true', help=(
                'list all active sites'))
    group.add_argument(
            '--job', dest='job', metavar='JOB_ID', help=(
                'list sites for a particular job'))
    group.add_argument(
            '--jobless', dest='jobless', action='store_true', help=(
                'list all jobless sites'))
    group.add_argument(
            '--site', dest='site', metavar='SITE_ID', help=(
                'list only the specified site'))
    group.add_argument(
            '--all', dest='all', action='store_true', help=(
                'list all sites'))
    add_rethinkdb_options(arg_parser)
    add_common_options(arg_parser, argv)

    args = arg_parser.parse_args(args=argv[1:])
    configure_logging(args)

    rr = rethinker(args)

    reql = rr.table('sites')
    if args.job:
        try:
            job_id = int(args.job)
        except ValueError:
            job_id = args.job
        reql = reql.get_all(job_id, index='job_id')
    elif args.jobless:
        reql = reql.filter(~r.row.has_fields('job_id'))
    elif args.active:
        reql = reql.between(
                ['ACTIVE', r.minval], ['ACTIVE', r.maxval],
                index='sites_last_disclaimed')
    elif args.site:
        reql = reql.get_all(args.site)
    logging.debug('querying rethinkdb: %s', reql)
    results = reql.run()
    if args.yaml:
        yaml.dump_all(
                results, stream=sys.stdout, explicit_start=True,
                default_flow_style=False)
    else:
        for result in results:
            print(json.dumps(result, cls=Jsonner, indent=2)) 

def brozzler_list_jobs(argv=None):
    argv = argv or sys.argv
    arg_parser = argparse.ArgumentParser(
            prog=os.path.basename(argv[0]),
            formatter_class=BetterArgumentDefaultsHelpFormatter)
    arg_parser.add_argument(
            '--yaml', dest='yaml', action='store_true', help=(
                'yaml output (default is json)'))
    group = arg_parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
            '--active', dest='active', action='store_true', help=(
                'list active jobs'))
    group.add_argument(
            '--all', dest='all', action='store_true', help=(
                'list all jobs'))
    group.add_argument(
            '--job', dest='job', metavar='JOB_ID', help=(
                'list only the specified job'))
    add_rethinkdb_options(arg_parser)
    add_common_options(arg_parser, argv)

    args = arg_parser.parse_args(args=argv[1:])
    configure_logging(args)

    rr = rethinker(args)
    if args.job is not None:
        try:
            job_id = int(args.job)
        except ValueError:
            job_id = args.job
        reql = rr.table('jobs').get(job_id)
        logging.debug('querying rethinkdb: %s', reql)
        result = reql.run()
        if result:
            results = [reql.run()]
        else:
            logging.error('no such job with id %r', job_id)
            sys.exit(1)
    else:
        reql = rr.table('jobs').order_by('id')
        if args.active:
            reql = reql.filter({'status': 'ACTIVE'})
        logging.debug('querying rethinkdb: %s', reql)
        results = reql.run()
    if args.yaml:
        yaml.dump_all(
                results, stream=sys.stdout, explicit_start=True,
                default_flow_style=False)
    else:
        for result in results:
            print(json.dumps(result, cls=Jsonner, indent=2))