Python yaml.SafeLoader() Examples

def __read_settings(self, yaml_file):
        if yaml_file is None:
            yaml_file = os.path.join(ciftify.config.find_ciftify_global(),
                    'ciftify_workflow_settings.yaml')
        if not os.path.exists(yaml_file):
            logger.critical("Settings yaml file {} does not exist"
                "".format(yaml_file))
            sys.exit(1)

        try:
            with open(yaml_file, 'r') as yaml_stream:
                config = yaml.load(yaml_stream, Loader=yaml.SafeLoader)
        except:
            logger.critical("Cannot read yaml config file {}, check formatting."
                    "".format(yaml_file))
            sys.exit(1)

        return config 

def __init__(self, filename, requests_and_responses=False):
        self._filename = filename
        self._requests_and_responses = requests_and_responses
        json_ext = '.json'
        yaml_ext = '.yaml'
        if filename[-len(json_ext):] == json_ext:
            self._scan = json.loads(open(filename).read().decode('utf-8-sig'), object_pairs_hook=UnsortableOrderedDict)
        elif filename[-len(yaml_ext):] == yaml_ext:
            self._scan = yaml_load(open(filename).read(), yaml.SafeLoader, UnsortableOrderedDict)
        else:
            # xml
            #self._xml = etree.parse(filename)
            etree_parser = etree.XMLParser(huge_tree=True)
            self._xml = etree.parse(filename, parser=etree_parser)
            root = self._xml.getroot()
            if root.tag == 'Sessions':
                self._webinspect_import()
            elif root.tag == 'issues':
                self._burp_import()
            elif root.tag == 'items':
                self._burp_items_import()
            else:
                raise Exception('Unknown scan format!') 

def grep_trace_entries(tracefile: str, job, scope, job_id=None):
        "All trace entries for the specified job_id or, if unspecified, the last job_id"
        if not job_id:
            entries = Trace.grep_entries(
                tracefile=tracefile,
                conjunctions=[f"job: {job}", f"scope: {scope}"],
                raw=True,
            )
            if not entries:
                return [], dict()
            job_id = yaml.load(entries[-1], Loader=yaml.SafeLoader).get("job_id")

        entries = Trace.grep_entries(
            tracefile=tracefile, conjunctions=[f"job_id: {job_id}", f"scope: {scope}"],
        )
        return entries 

def __read_mode(self, mode):
        logger = logging.getLogger(__name__)
        ciftify_data = config.find_ciftify_global()
        qc_settings = os.path.join(ciftify_data, 'qc_modes.yaml')
        try:
            with open(qc_settings, 'r') as qc_stream:
                qc_modes = yaml.load(qc_stream, Loader=yaml.SafeLoader)
        except:
            logger.error("Cannot read qc_modes file: {}".format(qc_settings))
            sys.exit(1)
        try:
            settings = qc_modes[mode]
        except KeyError:
            logger.error("qc_modes file {} does not define mode {}"
                    "".format(qc_settings, mode))
            sys.exit(1)
        return settings 

def test_secret_key_settings(self, mock_consul):
        """
        Test the YAML settings returned by SecretKeyInstanceMixin.
        """
        instance = OpenEdXInstanceFactory()
        secret_key_settings = yaml.load(instance.get_secret_key_settings(), Loader=yaml.SafeLoader)

        # Test that all keys are hex-encoded strings,
        # except for the JWK keys, wich must be valid JSON strings
        for secret_key_name in secret_key_settings.keys():
            if secret_key_name not in JWK_SET_KEY_NAMES:
                codecs.decode(
                    secret_key_settings[secret_key_name],
                    "hex"
                )
            else:
                json.loads(secret_key_settings[secret_key_name])

        # Make sure all independent secret keys are all different
        independent_secrets = set(secret_key_settings[var] for var in OPENEDX_SECRET_KEYS)
        self.assertEqual(len(independent_secrets), len(OPENEDX_SECRET_KEYS))

        # Verify that API client keys are set to the matching server key.
        for to_var, from_var in OPENEDX_SHARED_KEYS.items():
            self.assertEqual(secret_key_settings[to_var], secret_key_settings[from_var]) 

def __init__(self, folder: Optional[str] = None, load_default=True):
        """Initialize with the default configuration"""
        if load_default:
            import kge
            from kge.misc import filename_in_module

            with open(filename_in_module(kge, "config-default.yaml"), "r") as file:
                self.options: Dict[str, Any] = yaml.load(file, Loader=yaml.SafeLoader)
        else:
            self.options = {}

        self.folder = folder  # main folder (config file, checkpoints, ...)
        self.log_folder: Optional[str] = (
            None  # None means use self.folder; used for kge.log, trace.yaml
        )
        self.log_prefix: str = None

    # -- ACCESS METHODS ---------------------------------------------------------------- 

def parse(post_path):
    """Parses the metadata and content from the provided post.

    Arguments:
        post_path {str} -- The absolute path to the Markdown post
    """
    raw_yaml = ''
    markdown = ''
    in_yaml = True
    with open(post_path, 'r') as post:
        for line in post.readlines():
            # Check if this is the ending tag
            if line.strip() == YAML_BOUNDARY:
                if in_yaml and raw_yaml:
                    in_yaml = False
                    continue
            if in_yaml:
                raw_yaml += line
            else:
                markdown += line
    front_matter = yaml.load(raw_yaml, Loader=yaml.SafeLoader)
    markdown = markdown.strip()
    return front_matter, markdown 

def test_input_files(files, test_type, tmpdir):
    """ Test for access to Sample input files. """
    file_text = " ".join(files)
    sample_data = {SAMPLE_NAME_COLNAME: "test-sample",
                   DATA_SOURCE_COLNAME: file_text}
    s = Sample(sample_data)
    assert file_text == s.data_source
    assert files == s.input_file_paths
    if test_type == "to_disk":
        path_sample_file = tmpdir.join("test-sample.yaml").strpath
        s.to_yaml(path_sample_file)
        print("Sample items: {}".format(s.items()))
        with open(path_sample_file) as sf:
            reloaded_sample_data = yaml.load(sf, SafeLoader)
        print("reloaded keys: {}".format(list(reloaded_sample_data.keys())))
        try:
            s_reloaded = Sample(reloaded_sample_data)
        except Exception:
            with open(path_sample_file) as sf:
                print("LINES (below):\n{}".format("".join(sf.readlines())))
            raise
        assert files == s_reloaded.input_file_paths 

def handle(self, request, data):
        try:
            toscal = yaml.load(data['tosca'], Loader=yaml.SafeLoader)
            vnfd_name = data['name']
            vnfd_description = data['description']
            tosca_arg = {'vnfd': {'name': vnfd_name,
                                  'description': vnfd_description,
                                  'attributes': {'vnfd': toscal}}}
            vnfd_instance = api.tacker.create_vnfd(request, tosca_arg)
            messages.success(request,
                             _('VNF Catalog entry %s has been created.') %
                             vnfd_instance['vnfd']['name'])
            return toscal
        except Exception as e:
            msg = _('Unable to create TOSCA. %s')
            msg %= e.message.split('Failed validating', 1)[0]
            exceptions.handle(request, message=msg)
            return False 

def _assert_field_values(
            self,
            instance,
            sub_domain,
            configuration_version=SANDBOX_DEFAULTS["configuration_version"],
            openedx_release=SANDBOX_DEFAULTS["openedx_release"],
            configuration_extra_settings=SANDBOX_DEFAULTS["configuration_extra_settings"],
            openstack_server_flavor=SANDBOX_DEFAULTS["openstack_server_flavor"],
    ):
        """
        Assert that field values of `instance` match expected values
        """
        self.assertEqual(instance.internal_lms_domain, '{}.example.com'.format(sub_domain))
        self.assertEqual(instance.internal_lms_preview_domain, 'preview.{}.example.com'.format(sub_domain))
        self.assertEqual(instance.internal_studio_domain, 'studio.{}.example.com'.format(sub_domain))
        self.assertEqual(instance.configuration_version, configuration_version)
        self.assertEqual(instance.openedx_release, openedx_release)
        extra_settings = yaml.load(instance.configuration_extra_settings, Loader=yaml.SafeLoader)
        expected_extra_settings = yaml.load(configuration_extra_settings, Loader=yaml.SafeLoader)
        self.assertEqual(extra_settings, expected_extra_settings) 

def importyaml(connection,metadata,sourcePath):
    eveGraphics = Table('eveGraphics',metadata)
    print "Importing Graphics"
    print "opening Yaml"
    with open(os.path.join(sourcePath,'fsd','graphicIDs.yaml'),'r') as yamlstream:
        print "importing"
        trans = connection.begin()
        graphics=load(yamlstream,Loader=SafeLoader)
        print "Yaml Processed into memory"
        for graphic in graphics:
            connection.execute(eveGraphics.insert(),
                            graphicID=graphic,
                            sofFactionName=graphics[graphic].get('sofFactionName',''),
                            graphicFile=graphics[graphic].get('graphicFile',''),
                            sofHullName=graphics[graphic].get('sofHullName',''),
                            sofRaceName=graphics[graphic].get('sofRaceName',''),
                            description=graphics[graphic].get('description',''))
    trans.commit() 

def importyaml(connection,metadata,sourcePath,language='en'):
    invGroups = Table('invGroups',metadata)
    trnTranslations = Table('trnTranslations',metadata)
    print "Importing Groups"
    print "opening Yaml"
    with open(os.path.join(sourcePath,'fsd','groupIDs.yaml'),'r') as yamlstream:
        trans = connection.begin()
        groupids=load(yamlstream,Loader=SafeLoader)
        print "Yaml Processed into memory"
        for groupid in groupids:
            connection.execute(invGroups.insert(),
                            groupID=groupid,
                            categoryID=groupids[groupid].get('categoryID',0),
                            groupName=groupids[groupid].get('name',{}).get(language,'').decode('utf-8'),
                            iconID=groupids[groupid].get('iconID'),
                            useBasePrice=groupids[groupid].get('useBasePrice'),
                            anchored=groupids[groupid].get('anchored',0),
                            anchorable=groupids[groupid].get('anchorable',0),
                            fittableNonSingleton=groupids[groupid].get('fittableNonSingleton',0),
                            published=groupids[groupid].get('published',0))
            if (groupids[groupid].has_key('name')):
                for lang in groupids[groupid]['name']:
                    connection.execute(trnTranslations.insert(),tcID=7,keyID=groupid,languageID=lang,text=groupids[groupid]['name'][lang].decode('utf-8'));
    trans.commit() 

def from_yaml(cls, filepath):
        """
        Generate the configuration dictionary from yaml file
        :param filepath: config file path
        :return: None
        """
        # Check if the filepath provided exists
        if not os.path.isfile(filepath):
            raise FileNotFoundError("Invalid path for the configuration file")

        # Parse the configuration and merge it in dict
        with open(filepath, 'r') as yamlfile:
            try:
                # Getting config from the file
                config = yaml.load(yamlfile, Loader=yaml.SafeLoader)
                # Merging the dictionaries and getting result
                cls.cfg = {**cls.cfg, **config}
            except yaml.YAMLError as e:
                print(e) 

def test_serve_local_code_no_env(tmpdir, sagemaker_session):
    with patch(
        "sagemaker.local.image._SageMakerContainer._create_tmp_folder",
        return_value=str(tmpdir.mkdir("container-root")),
    ):
        image = "my-image"
        sagemaker_container = _SageMakerContainer(
            "local", 1, image, sagemaker_session=sagemaker_session
        )
        sagemaker_container.serve("/some/model/path", {})
        docker_compose_file = os.path.join(
            sagemaker_container.container_root, "docker-compose.yaml"
        )

        with open(docker_compose_file, "r") as f:
            config = yaml.load(f, Loader=yaml.SafeLoader)

            for h in sagemaker_container.hosts:
                assert config["services"][h]["image"] == image
                assert config["services"][h]["command"] == "serve" 

def importyaml(connection,metadata,sourcePath,language='en'):
    print "Importing dogma attribute categories"
    dgmAttributeCategories = Table('dgmAttributeCategories',metadata)
    
    print "opening Yaml"
        
    trans = connection.begin()
    with open(os.path.join(sourcePath,'fsd','dogmaAttributeCategories.yaml'),'r') as yamlstream:
        print "importing"
        dogmaAttributeCategories=load(yamlstream,Loader=SafeLoader)
        print "Yaml Processed into memory"
        for dogmaAttributeCategoryID in dogmaAttributeCategories:
          attribute = dogmaAttributeCategories[dogmaAttributeCategoryID]
          connection.execute(dgmAttributeCategories.insert(),
                             categoryID=dogmaAttributeCategoryID,
                             categoryName=attribute['name'],
                             categoryDescription=attribute['description']
                )
    trans.commit() 

def GUI():
    wx_app = wx.App(redirect=True) # redirect in wxpython 3.0 defaults to False
    #wx_app = wx.App(redirect=False)
    #YamledWindow(content='../workbench/x.yaml')
    #YamledWindow(content='../workbench/y.yaml')
    #YamledWindow(content='../workbench/_yamled_dies.yaml')
    #YamledWindow(content='../workbench/yamled/sample-1.yaml')
    #YamledWindow(content='../workbench/yamled/pt.yaml')
    #YamledWindow(content=yaml_load(open('../workbench/yamled/burp-state-1-report.yaml').read(), yaml.SafeLoader, UnsortableOrderedDict))
    #YamledWindow(content='../workbench/yamled/asdf.yaml')
    #YamledWindow(content='../workbench/yamled/asdfgh.yaml')
    #YamledWindow(content='../workbench/yamled/burp-state-1-report.yaml')
    #YamledWindow(content='../workbench/yamled/_export_webinspect.yaml')
    #YamledWindow(content='../workbench/yamled/midsized.yaml')
    YamledWindow()
    #YamledWindow(content='../workbench/xss-2-intruder-items.yaml')
    wx_app.MainLoop() 

def Deserializer(stream_or_string, **options):
    """
    Deserialize a stream or string of YAML data.
    """
    if isinstance(stream_or_string, bytes):
        stream_or_string = stream_or_string.decode('utf-8')
    if isinstance(stream_or_string, six.string_types):
        stream = StringIO(stream_or_string)
    else:
        stream = stream_or_string
    try:
        for obj in PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options):
            yield obj
    except GeneratorExit:
        raise
    except Exception as e:
        # Map to deserializer error
        six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2]) 

def parse_args(parser):
    args = parser.parse_args()
    if args.config_file:
        loader = yaml.SafeLoader
        loader.add_implicit_resolver(
            u'tag:yaml.org,2002:float',
            re.compile(u'''^(?:
        [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
        |\\.[0-9_]+(?:[eE][-+][0-9]+)?
        |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
        |[-+]?\\.(?:inf|Inf|INF)
        |\\.(?:nan|NaN|NAN))$''', re.X),
            list(u'-+0123456789.'))
        data = yaml.load(args.config_file, Loader=loader)
        delattr(args, 'config_file')
        arg_dict = args.__dict__
        # print(len(list(arg_dict.keys())))
        # print(len(list(data.keys())))
        for key, value in arg_dict.items():
            default_arg = parser.get_default(key)
            if arg_dict[key] == default_arg and key in data:
                arg_dict[key] = data[key]
    return args 

def _input_json_from_file(path):
        try:
            with open(path, encoding="utf-8") as file:
                return load(file)
        except IOError as e:
            raise RuntimeError(
                "File '{}' cannot be opened:\n{}".format(path, e)
            )
        except ValueError as e:
            try:
                with open(path, encoding="utf-8") as file:
                    return load_yaml(file, Loader=SafeLoader)
            except ValueError:
                raise RuntimeError(
                    "File '{}' is not valid JSON or YAML:\n{}".format(path, e)
                ) 

def gtfobins(bin_name: str):
    """Search binaries from GTFOBins within command line

    Arguments:
        bin_name {[type]} -- Name of the binary to get info about

    """

    bins = get_bins()

    if bin_name in bins:
        r = requests.get(RAW_URL.format(bin_name)).text
        data = list(yaml.load_all(r, Loader=yaml.SafeLoader))[0]

        parse(data)
    else:
        print(colors("[!] Binary not found on GTFObins: ", 91)) 

def lolbas(name: str):
    """Search binaries from LOLBAS within command line

    Arguments:
        name {[type]} -- Name of the exe to get info about

    Keyword Arguments:
        cmd {str} -- get only the code section (default: {False})
    """

    exes = get_exe()
    if name in exes.keys():
        url = RAW_URL + exes[name] + '.md'
        r = requests.get(url).text
        data = list(yaml.load_all(r, Loader=yaml.SafeLoader))[0]
        parse(data)
    else:
        print(colors("[!] Binary not found on LOLBAS", 91))
        #TODO: Match user input and make suggestion for search
        print(colors("[!] Make sure to provide name with proper extension", 91)) 

def read(self, config_file):
        cfg = None
        if isinstance(config_file, dict):
            cfg = config_file.get('atomicpuppy')
        elif isinstance(config_file, str):
            with open(config_file) as file:
                cfg = yaml.load(file, yaml.SafeLoader).get('atomicpuppy')
        else:
            cfg = yaml.load(config_file, yaml.SafeLoader).get('atomicpuppy')


        streams = []
        instance = cfg.get('instance') or platform.node()
        for stream in cfg.get("streams"):
            streams.append(stream)
        ctr = self._make_counter(cfg, instance)
        return SubscriptionConfig(streams=streams,
                                  counter_factory=ctr,
                                  instance_name=instance,
                                  host=cfg.get("host") or 'localhost',
                                  port=cfg.get("port") or 2113,
                                  timeout=cfg.get("timeout") or 20,
                                  page_size=cfg.get("page_size") or 20) 

def make_loader(*tags_to_remove: str) -> Type[yaml.SafeLoader]:
    """Create a YAML loader, that doesn't parse specific tokens into Python objects."""
    cls: Type[yaml.SafeLoader] = type("YAMLLoader", (SafeLoader,), {})
    cls.yaml_implicit_resolvers = {
        key: [(tag, regexp) for tag, regexp in mapping if tag not in tags_to_remove]
        for key, mapping in cls.yaml_implicit_resolvers.copy().items()
    }

    # Fix pyyaml scientific notation parse bug
    # See PR: https://github.com/yaml/pyyaml/pull/174 for upstream fix
    cls.add_implicit_resolver(  # type: ignore
        "tag:yaml.org,2002:float",
        re.compile(
            r"""^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+]?[0-9]+)?
                       |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
                       |\.[0-9_]+(?:[eE][-+]?[0-9]+)?
                       |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
                       |[-+]?\.(?:inf|Inf|INF)
                       |\.(?:nan|NaN|NAN))$""",
            re.X,
        ),
        list("-+0123456789."),
    )

    return cls 

def test_serve(tmpdir, sagemaker_session):
    with patch(
        "sagemaker.local.image._SageMakerContainer._create_tmp_folder",
        return_value=str(tmpdir.mkdir("container-root")),
    ):
        image = "my-image"
        sagemaker_container = _SageMakerContainer(
            "local", 1, image, sagemaker_session=sagemaker_session
        )
        environment = {"env1": 1, "env2": "b", "SAGEMAKER_SUBMIT_DIRECTORY": "s3://some/path"}

        sagemaker_container.serve("/some/model/path", environment)
        docker_compose_file = os.path.join(
            sagemaker_container.container_root, "docker-compose.yaml"
        )

        with open(docker_compose_file, "r") as f:
            config = yaml.load(f, Loader=yaml.SafeLoader)

            for h in sagemaker_container.hosts:
                assert config["services"][h]["image"] == image
                assert config["services"][h]["command"] == "serve" 

def importyaml(connection,metadata,sourcePath):

    print "Importing BSD Tables"

    files=glob.glob(os.path.join(sourcePath,'bsd','*.yaml'))
    for file in files:
        head, tail = os.path.split(file)
        tablename=tail.split('.')[0]
        print tablename
        tablevar = Table(tablename,metadata)
        print "Importing {}".format(file)
        print "Opening Yaml"
        trans = connection.begin()
        with open(file,'r') as yamlstream:
            rows=load(yamlstream,Loader=SafeLoader)
            print "Yaml Processed into memory"
            for row in rows:
                connection.execute(tablevar.insert().values(row))
        trans.commit() 

def test_create_from_pr_and_watchedfork_values(self):
        """
        Create an instance from a pull request, and check that the default values from the watched fork are used.
        """
        pr = PRFactory()
        _, organization = make_user_and_organization()
        watched_fork = WatchedForkFactory(
            fork=pr.fork_name,
            organization=organization,
            configuration_source_repo_url='https://github.com/open-craft/configuration-fromwatchedfork',
            configuration_version='named-release/elder-fromwatchedfork',
            configuration_extra_settings=textwrap.dedent("""\
                PHRASE: "Hello"
                """),
            openedx_release='ginkgo.8',
        )
        with patch(
                'instance.models.openedx_instance.OpenEdXInstance._write_metadata_to_consul',
                return_value=(1, True)
        ):
            instance, created = WatchedPullRequest.objects.get_or_create_from_pr(pr, watched_fork)
        self.assertTrue(created)

        self.assertEqual(instance.configuration_source_repo_url,
                         'https://github.com/open-craft/configuration-fromwatchedfork')
        self.assertEqual(instance.configuration_version,
                         'named-release/elder-fromwatchedfork')
        self.assertEqual(instance.openedx_release, 'ginkgo.8')
        self.assertEqual(yaml.load(instance.configuration_extra_settings, Loader=yaml.SafeLoader), {'PHRASE': 'Hello'}) 

def get_extra_setting(self, name, default=None):
        """
        Return the setting given by "name" from extra_settings.

        The name may be a dot-separated path to retrieve nested settings.
        """
        extra_settings_dict = yaml.load(self.extra_settings, Loader=yaml.SafeLoader) or {}
        try:
            return functools.reduce(operator.getitem, name.split('.'), extra_settings_dict)
        except KeyError:
            return default 

def decode_file(filename):
    with open(filename) as f:
        if filename.endswith('.json'):
            return json.load(f)
        else:
            return yaml.load(f, Loader=SafeLoader) 

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('template')
    parser.add_argument('datafile')

    args = parser.parse_args()

    tplfile = f'{args.template}.tpl.yml'
    path = pathlib.Path(__file__).parent / tplfile

    if not path.exists():
        die(f'template does not exist: {tplfile}')

    with open(path) as f:
        tpl = env.from_string(f.read())

    datapath = pathlib.Path(__file__).parent / args.datafile

    if not datapath.exists():
        die(f'data file does not exist: {args.datafile}')

    with open(datapath) as f:
        data = yaml.load(f, Loader=yaml.SafeLoader)

    output = tpl.render(**data)

    target = (
        pathlib.Path(__file__).parent.parent
        / 'workflows'
        / f'{args.template}.yml'
    )
    with open(target, 'w') as f:
        print(output, file=f) 

def safe_load(stream, enforce_order=True):
    return load(stream, SortEnforcingLoader if enforce_order else SafeLoader)