Python yaml.add_constructor() Examples

def get_config(dir='config/config.yaml'):
    # add direction join function when parse the yaml file
    def join(loader, node):
        seq = loader.construct_sequence(node)
        return os.path.sep.join(seq)

    # add string concatenation function when parse the yaml file
    def concat(loader, node):
        seq = loader.construct_sequence(node)
        seq = [str(tmp) for tmp in seq]
        return ''.join(seq)

    yaml.add_constructor('!join', join)
    yaml.add_constructor('!concat', concat)
    with open(dir, 'r') as f:
        cfg = yaml.load(f)

    check_dirs(cfg)

    return cfg 

def get_config(dir):
    # add direction join function when parse the yaml file
    def join(loader, node):
        seq = loader.construct_sequence(node)
        return os.path.sep.join(seq)

    # add string concatenation function when parse the yaml file
    def concat(loader, node):
        seq = loader.construct_sequence(node)
        return ''.join(seq)

    yaml.add_constructor('!join', join)
    yaml.add_constructor('!concat', concat)
    with open(dir, 'r') as f:
        cfg = yaml.load(f)

    return cfg 

def initialize():
    """
    Initialize the configuration system by installing YAML handlers.
    Automatically done on first call to load() specified in this file.
    """
    global is_initialized

    # Add the custom multi-constructor
    yaml.add_multi_constructor('!obj:', multi_constructor_obj)
    yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)
    yaml.add_multi_constructor('!import:', multi_constructor_import)

    yaml.add_constructor('!import', constructor_import)
    yaml.add_constructor("!float", constructor_float)

    pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP)
    yaml.add_implicit_resolver('!float', pattern)

    is_initialized = True


###############################################################################
# Callbacks used by PyYAML 

def parse_minionqc_report(self, s_name, f):
        '''
        Parses minionqc's 'summary.yaml' report file for results.
        Uses only the "All reads" stats. Ignores "Q>=x" part.
        '''
        try:
            # Parsing as OrderedDict is slightly messier with YAML
            # http://stackoverflow.com/a/21048064/713980
            def dict_constructor(loader, node):
                return OrderedDict(loader.construct_pairs(node))
            yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_constructor)
            summary_dict = yaml.safe_load(f)
        except Exception as e:
            log.error("Error parsing MinIONQC input file: {}".format(f))
            return

        # Do a deep copy as dicts are immutable
        self.minionqc_raw_data[s_name] = copy.deepcopy(summary_dict)

        # get q value threshold used for reads
        q_threshold = None
        for k in summary_dict.keys():
            if k.startswith('Q>='):
                q_threshold = k

        data_dict = {}
        data_dict['all'] = summary_dict['All reads']        # all reads
        data_dict['q_filt'] = summary_dict[q_threshold]     # quality filtered reads

        for q_key in ['all', 'q_filt']:
            for key_1 in ['reads', 'gigabases']:
                for key_2 in data_dict[q_key][key_1]:
                    new_key = '{} {}'.format(key_1, key_2)
                    data_dict[q_key][new_key] = data_dict[q_key][key_1][key_2]
                data_dict[q_key].pop(key_1)    # removes key after flattening

        self.minionqc_data[s_name] = data_dict['all']     # stats for all reads
        self.qfilt_data[s_name] = data_dict['q_filt']     # stats for q-filtered reads
        self.q_threshold_list.add(q_threshold)            # quality threshold used in this file 

def _config_loader():
    envvar_matcher = re.compile(r"\${([^}^{]+)\}")

    def envvar_constructor(_loader, node):  # pragma: no cover
        """Extract the matched value, expand env variable, and replace the match."""
        node_value = node.value
        match = envvar_matcher.match(node_value)
        env_var = match.group()[2:-1]

        # check for defaults
        var_name, default_value = env_var.split(":")
        var_name = var_name.strip()
        default_value = default_value.strip()
        var_value = os.getenv(var_name, default_value)
        return var_value + node_value[match.end() :]

    yaml.add_implicit_resolver("!envvar", envvar_matcher, None, SafeLoader)
    yaml.add_constructor("!envvar", envvar_constructor, SafeLoader)


# TODO: instead of this, create custom loader and use it
#       by wrapping yaml.safe_load to use it 

def demo():
    constructor = CustodiaSimpleConstructor(
        'http+unix://%2E%2Fserver_socket/secrets'
    )
    constructor.client.headers['REMOTE_USER'] = 'user'

    # create entries
    try:
        c = constructor.client.list_container('test')
    except HTTPError:
        constructor.client.create_container('test')
        c = []
    if 'key' not in c:
        constructor.client.set_secret('test/key', 'secret password')

    yaml.add_constructor(CustodiaSimpleConstructor.yaml_tag,
                         constructor)
    yaml_str = 'password: !custodia/simple test/key'
    print(yaml_str)
    result = yaml.load(yaml_str)
    print(result) 

def constructor(tag=None, pattern=None):
    """Register custom constructor with pyyaml."""
    def decorator(f):
        if tag is None or f is tag:
            tag_ = '!{}'.format(f.__name__)
        else:
            tag_ = tag
        yaml.add_constructor(tag_, f)
        if pattern is not None:
            yaml.add_implicit_resolver(tag_, re.compile(pattern))
        return f
    if callable(tag):  # little convenience hack to avoid empty arg list
        return decorator(tag)
    return decorator 

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 

def register(cls):
        """ Register variable handling in YAML """
        if not cls.IS_LOADED:
            cls.IS_LOADED = True

            yaml.add_constructor('!param', Parameter.parameter_constructor, Loader=yaml.SafeLoader)
            yaml.add_constructor('!env', EnvironmentVariable.parameter_constructor, Loader=yaml.SafeLoader) 

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 

def __init__(self, environment_name, nova_descriptor_file=None):
        self._nova_descriptor_file = nova_descriptor_file or 'nova.yml'
        self._environment_name = environment_name
        self._environment = None
        self._codedeploy_app = None

        self.templates_used = dict()
        yaml.add_constructor("!include", yaml_include)

        with open(os.path.join(spec.__path__[0], 'nova_service_schema.yml'), 'r') as schemaYaml:
            schema = yaml.load(schemaYaml)

        v = Validator(schema)
        try:
            with open(self._nova_descriptor_file, 'r') as novaYaml:
                self.service_spec = yaml.safe_load(novaYaml)

            # Validate loaded dictionary
            valid = v.validate(self.service_spec)
            if not valid:
                raise NovaError("Invalid nova service descriptor file '%s': %s" % (self._nova_descriptor_file, v.errors))
            else:
                self.service = Service.load(self.service_spec)
                self.service_name = self.service.name
                self.service_port = self.service.port
                self.service_healthcheck_url = self.service.healthcheck_url
        except IOError:
            raise NovaError("No nova service descriptor found at '%s'" % self._nova_descriptor_file) 

def parse_yaml_config(yaml_path):
    """Parses a yaml config

    Args:
        yaml_path: path to yaml config

    Returns:
        config_obj: config converted to object
    """

    # Add check for duplicate keys in yaml
    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)

    with open(yaml_path, 'r') as yaml_file:
        config_dict = yaml.load(yaml_file)

    config_obj = config_dict_to_object(config_dict)
    config_obj.config_name = os.path.splitext(os.path.basename(yaml_path))[0]
    config_obj.exp_output_dir = monopsr.data_dir() + '/outputs/' + config_obj.config_name

    # Prepend data folder to paths
    paths_config = config_obj.train_config.paths_config
    if paths_config.checkpoint_dir is None:
        checkpoint_dir = config_obj.exp_output_dir + '/checkpoints'

        if not os.path.exists(checkpoint_dir):
            os.makedirs(checkpoint_dir)

        paths_config.checkpoint_dir = checkpoint_dir
    else:
        paths_config.checkpoint_dir = os.path.expanduser(paths_config.checkpoint_dir)

    paths_config.logdir = config_obj.exp_output_dir + '/logs'
    paths_config.pred_dir = config_obj.exp_output_dir + '/predictions'

    return config_obj 

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 

def load_ordered_config(config_path):
    """
      Loads the configuration in the same order as it's defined in yaml file,
      so that, while saving it in new format, order is maintained
      Args:
            config_path (str): Path to the configuration file
      Returns:
            config(dict): Returns the configurations in the defined ordered
    """

    #  To load data from yaml in ordered dict format
    _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG

    def dict_representer(dumper, data):
        return dumper.represent_mapping(_mapping_tag, iteritems(data))

    def dict_constructor(loader, node):
        return collections.OrderedDict(loader.construct_pairs(node))

    yaml.add_representer(collections.OrderedDict, dict_representer)
    yaml.add_constructor(_mapping_tag, dict_constructor)

    #  format the output to print a blank scalar rather than null
    def represent_none(self, _):
        return self.represent_scalar('tag:yaml.org,2002:null', u'')

    yaml.add_representer(type(None), represent_none)

    # read input from home directory for pull requests
    with open(config_path, 'r') as f:
        config = yaml.safe_load(f)
    return config 

def setup_yaml():
    _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG

    def dict_representer(dumper, data):
        return dumper.represent_dict(data.items())

    def dict_constructor(loader, node):
        return collections.OrderedDict(loader.construct_pairs(node))

    yaml.SafeDumper.add_representer(collections.OrderedDict, dict_representer)
    yaml.add_constructor(_mapping_tag, dict_constructor) 

def register(cls):
        yaml.add_constructor(cls.tag, cls.constructor)
        yaml.add_representer(cls, cls.representer) 

def register_tag(tag, classpath):
    yaml.add_constructor(u'!'+tag, metaloader(classpath))
    yaml.add_constructor(u'tag:nltk.org,2011:'+tag,
                         metaloader(classpath)) 

def read(self, filepath):
        stream = False

        # Setup environment variables replacement
        def env_var_single_replace(match):
            return os.environ[match.group(1)] if match.group(1) in os.environ else match.group()

        def env_var_multi_replacer(loader, node):
            value = loader.construct_scalar(node)

            return re.sub(ENV_VAR_REPLACER_PATTERN, env_var_single_replace, value)

        yaml.add_implicit_resolver("!envvarreplacer", ENV_VAR_MATCHER_PATTERN)
        yaml.add_constructor('!envvarreplacer', env_var_multi_replacer)

        # Read file
        try:
            stream = open(filepath, "r")
            self.config = yaml.load(stream)

            # Handle an empty configuration file
            if not self.config:
                self.config = {}
        finally:
            if stream:
                stream.close() 

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 

def load(s, encoding='utf-8'):

    # override to set output encoding
    def construct_yaml_str(loader, node):
        value = loader.construct_scalar(node)

        if encoding is not None:
            return value.encode(encoding)

        return value

    yaml.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)

    return yaml.load(s) 

def setup_yaml():
    _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
    yaml.add_representer(collections.OrderedDict, dict_representer)
    yaml.add_representer(os._Environ, dict_representer)
    yaml.add_representer(unicode, unicode_representer)
    yaml.add_constructor(_mapping_tag, dict_constructor) 

def testIfTableIsAMap(self):

        from yaml.constructor import ConstructorError

        try:
            from yaml import CLoader as Loader
        except ImportError:
            from yaml import Loader

        # Credit https://gist.github.com/pypt/94d747fe5180851196eb
        def no_duplicates_constructor(loader, node, deep=False):
            """Check for duplicate keys."""

            mapping = {}
            for key_node, value_node in node.value:
                key = loader.construct_object(key_node, deep=deep)
                value = loader.construct_object(value_node, deep=deep)
                if key in mapping:
                    raise ConstructorError(
                        "while constructing a mapping", node.start_mark,
                        "found duplicate key (%s)" % key, key_node.start_mark)
                mapping[key] = value

            return loader.construct_mapping(node, deep)

        yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)

        file_path = '../translationtable/GLOBAL_TERMS.yaml'
        if os.path.exists(os.path.join(os.path.dirname(__file__), file_path)):
            tt_file = open(os.path.join(os.path.dirname(__file__), file_path), 'r')
            try:
                translation_table = yaml.safe_load(tt_file)
            except yaml.constructor.ConstructorError as e:
                tt_file.close()
                self.assertTrue(False)
                print(e)
            tt_file.close()