Python utils.Config() Examples

The following are 13 code examples of utils.Config(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module utils , or try the search function .
Example #1
Source File: train.py    From FlexTensor with MIT License 5 votes vote down vote up 
def load_data(train_file, test_file, eval_dev=-1):
    for filename in [train_file, test_file]:
        if filename is None:
            continue
        if not (os.path.exists(filename) and os.path.isfile(filename)):
            raise RuntimeError("File not found when loading data from %s" % filename)

    def _load(filename):
        if filename is None:
            return []
        ret = []
        with open(filename, "r") as fin:
            for line in fin:
                if line:
                    key, string = line.split(":", 1)
                    op, _, shape_str, target_str = key.split("_")
                    shape = [int(x) for x in shape_str[1:-1].split(", ")]
                    target, dev_id_str = target_str[:-1].split("(")
                    dev_id = int(dev_id_str) if eval_dev < 0 else eval_dev
                    config = json.loads(string)
                    ret.append(DataItem(
                        op=op, 
                        shape=shape, 
                        target=TargetItem(target=target, dev_id=dev_id), 
                        config=utils.Config(config[0], config[1]))
                    )
        return ret
    
    return _load(train_file), _load(test_file)
Example #2
Source File: model.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up 
def get_estimator(**kwargs):
  """Construct an estimator."""
  cfg = utils.Config(kwargs)

  if cfg.tpu.get('name'):
    tf.logging.info('Using cluster resolver.')
    cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
        cfg.tpu.name, zone=cfg.tpu.zone, project=cfg.tpu.gcp_project)
    master = None
  else:
    cluster_resolver = None
    master = cfg.master

  tf.logging.info('Config:\n %s' % cfg)
  if cfg.tpu.enable:
    if not cfg.steps_per_epoch:
      raise ValueError('steps_per_epoch must be nonzero on TPU.')
    exp = tf.contrib.tpu.TPUEstimator(
        model_fn=model_fn,
        config=tf.contrib.tpu.RunConfig(
            cluster=cluster_resolver,
            master=master,
            model_dir=cfg.model_dir,
            tpu_config=tf.contrib.tpu.TPUConfig(
                iterations_per_loop=cfg.steps_per_epoch)),
        use_tpu=True,
        eval_on_tpu=False,
        # TPU requires these args, but they are ignored inside the input
        # function, which directly get train_batch_size or eval_batch_size.
        train_batch_size=cfg.dataset.train_batch_size,
        eval_batch_size=cfg.dataset.eval_batch_size,
        params=cfg,
    )
  else:
    exp = tf.estimator.Estimator(
        model_fn=model_fn, model_dir=cfg.model_dir, params=cfg)

  return exp
Example #3
Source File: model.py    From tpu_models with Apache License 2.0 5 votes vote down vote up 
def get_estimator(**kwargs):
  """Construct an estimator."""
  cfg = utils.Config(kwargs)

  if cfg.tpu.get('name'):
    tf.logging.info('Using cluster resolver.')
    cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
        cfg.tpu.name, zone=cfg.tpu.zone, project=cfg.tpu.gcp_project)
    master = None
  else:
    cluster_resolver = None
    master = cfg.master

  tf.logging.info('Config:\n %s' % cfg)
  if cfg.tpu.enable:
    if not cfg.steps_per_epoch:
      raise ValueError('steps_per_epoch must be nonzero on TPU.')
    exp = tf.contrib.tpu.TPUEstimator(
        model_fn=model_fn,
        config=tf.contrib.tpu.RunConfig(
            cluster=cluster_resolver,
            master=master,
            model_dir=cfg.model_dir,
            tpu_config=tf.contrib.tpu.TPUConfig(
                iterations_per_loop=cfg.steps_per_epoch)),
        use_tpu=True,
        eval_on_tpu=False,
        # TPU requires these args, but they are ignored inside the input
        # function, which directly get train_batch_size or eval_batch_size.
        train_batch_size=cfg.dataset.train_batch_size,
        eval_batch_size=cfg.dataset.eval_batch_size,
        params=cfg,
    )
  else:
    exp = tf.estimator.Estimator(
        model_fn=model_fn, model_dir=cfg.model_dir, params=cfg)

  return exp
Example #4
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)
    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    tokenizer = get_tokenizer(dataset_config, model_config)

    # model (restore)
    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = SenCNN(num_classes=model_config.num_classes, vocab=tokenizer.vocab)
    model.load_state_dict(checkpoint["model_state_dict"])

    # evaluation
    summary_manager = SummaryManager(exp_dir)
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, tokenizer.split_and_transform)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary = evaluate(model, dl, {"loss": nn.CrossEntropyLoss(), "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")
    print(f"loss: {summary['loss']:.3f}, acc: {summary['acc']:.2%}")
Example #5
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)
    ptr_config_info = Config(f"conf/pretrained/{model_config.type}.json")

    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
        f"_weight_decay_{args.weight_decay}"
    )

    preprocessor = get_preprocessor(ptr_config_info, model_config)

    with open(ptr_config_info.config, mode="r") as io:
        ptr_config = json.load(io)

    # model (restore)
    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint('best.tar')
    config = BertConfig()
    config.update(ptr_config)
    model = PairwiseClassifier(config, num_classes=model_config.num_classes, vocab=preprocessor.vocab)
    model.load_state_dict(checkpoint['model_state_dict'])

    # evaluation
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, preprocessor.preprocess)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4)
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    model.to(device)

    summary_manager = SummaryManager(exp_dir)
    summary = evaluate(model, dl, {'loss': nn.CrossEntropyLoss(), 'acc': acc}, device)

    summary_manager.load('summary.json')
    summary_manager.update({'{}'.format(args.data): summary})
    summary_manager.save('summary.json')

    print('loss: {:.3f}, acc: {:.2%}'.format(summary['loss'], summary['acc']))
Example #6
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)

    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    tokenizer = get_tokenizer(dataset_config)

    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = ConvRec(num_classes=model_config.num_classes, embedding_dim=model_config.embedding_dim,
                    hidden_dim=model_config.hidden_dim, vocab=tokenizer.vocab)
    model.load_state_dict(checkpoint["model_state_dict"])

    summary_manager = SummaryManager(exp_dir)
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, tokenizer.split_and_transform, min_length=model_config.min_length,
                pad_val=tokenizer.vocab.to_indices(' '))
    dl = DataLoader(ds, batch_size=args.batch_size, collate_fn=batchify)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary = evaluate(model, dl, {"loss": nn.CrossEntropyLoss(), "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")
    print(f"loss: {summary['loss']:.3f}, acc: {summary['acc']:.2%}")
Example #7
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)
    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    tokenizer = get_tokenizer(dataset_config, model_config)

    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = VDCNN(num_classes=model_config.num_classes, embedding_dim=model_config.embedding_dim,
                  k_max=model_config.k_max, vocab=tokenizer.vocab)
    model.load_state_dict(checkpoint["model_state_dict"])

    summary_manager = SummaryManager(exp_dir)
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, tokenizer.split_and_transform)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary = evaluate(model, dl, {"loss": nn.CrossEntropyLoss(), "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")
    print(f"loss: {summary['loss']:.3f}, acc: {summary['acc']:.2%}")
Example #8
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)

    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    tokenizer = get_tokenizer(dataset_config)

    # model (restore)
    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = SAN(num_classes=model_config.num_classes, lstm_hidden_dim=model_config.lstm_hidden_dim,
                da=model_config.da, r=model_config.r, hidden_dim=model_config.hidden_dim, vocab=tokenizer.vocab)
    model.load_state_dict(checkpoint["model_state_dict"])

    # evaluation
    summary_manager = SummaryManager(exp_dir)
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, tokenizer.split_and_transform)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4, collate_fn=batchify)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary = evaluate(model, dl, {"loss": nn.CrossEntropyLoss(), "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")
    print(f"loss: {summary['loss']:.3f}, acc: {summary['acc']:.2%}")
Example #9
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)
    ptr_config_info = Config(f"conf/pretrained/{model_config.type}.json")

    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
        f"_weight_decay_{args.weight_decay}"
    )

    preprocessor = get_preprocessor(ptr_config_info, model_config)

    with open(ptr_config_info.config, mode="r") as io:
        ptr_config = json.load(io)

    # model (restore)
    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint('best.tar')
    config = BertConfig()
    config.update(ptr_config)
    model = SentenceClassifier(config, num_classes=model_config.num_classes, vocab=preprocessor.vocab)
    model.load_state_dict(checkpoint['model_state_dict'])

    # evaluation
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, preprocessor.preprocess)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4)
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    model.to(device)

    summary_manager = SummaryManager(exp_dir)
    summary = evaluate(model, dl, {'loss': nn.CrossEntropyLoss(), 'acc': acc}, device)

    summary_manager.load('summary.json')
    summary_manager.update({'{}'.format(args.data): summary})
    summary_manager.save('summary.json')

    print('loss: {:.3f}, acc: {:.2%}'.format(summary['loss'], summary['acc']))
Example #10
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)

    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    preprocessor = get_preprocessor(dataset_config, coarse_split_fn=split_morphs, fine_split_fn=split_jamos)

    # model (restore)
    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = SAN(model_config.num_classes, preprocessor.coarse_vocab, preprocessor.fine_vocab,
                model_config.fine_embedding_dim, model_config.hidden_dim, model_config.multi_step,
                model_config.prediction_drop_ratio)
    model.load_state_dict(checkpoint["model_state_dict"])

    # evaluation
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, preprocessor.preprocess)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4, collate_fn=batchify)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary_manager = SummaryManager(exp_dir)
    summary = evaluate(model, dl, {"loss": log_loss, "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")

    print(f"loss: {summary['loss']:.3f}, acc: {summary['acc']:.2%}")
Example #11
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)
    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    tokenizer = get_tokenizer(dataset_config, model_config)

    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = CharCNN(num_classes=model_config.num_classes, embedding_dim=model_config.embedding_dim,
                    vocab=tokenizer.vocab)
    model.load_state_dict(checkpoint["model_state_dict"])

    summary_manager = SummaryManager(exp_dir)
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, tokenizer.split_and_transform)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary = evaluate(model, dl, {"loss": nn.CrossEntropyLoss(), "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")
    print(f"loss: {summary['loss']:.3f}, acc: {summary['acc']:.2%}")
Example #12
Source File: evaluate.py    From nlp_classification with MIT License 5 votes vote down vote up 
def main(args):
    dataset_config = Config(args.dataset_config)
    model_config = Config(args.model_config)

    exp_dir = Path("experiments") / model_config.type
    exp_dir = exp_dir.joinpath(
        f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}"
    )

    tokenizer = get_tokenizer(dataset_config, split_fn=split_morphs)

    # model (restore)
    checkpoint_manager = CheckpointManager(exp_dir)
    checkpoint = checkpoint_manager.load_checkpoint("best.tar")
    model = MaLSTM(num_classes=model_config.num_classes, hidden_dim=model_config.hidden_dim, vocab=tokenizer.vocab)
    model.load_state_dict(checkpoint["model_state_dict"])

    # evaluation
    filepath = getattr(dataset_config, args.data)
    ds = Corpus(filepath, tokenizer.split_and_transform)
    dl = DataLoader(ds, batch_size=args.batch_size, num_workers=4, collate_fn=batchify)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)

    summary_manager = SummaryManager(exp_dir)
    summary = evaluate(model, dl, {"loss": nn.CrossEntropyLoss(), "acc": acc}, device)

    summary_manager.load("summary.json")
    summary_manager.update({f"{args.data}": summary})
    summary_manager.save("summary.json")

    print("loss: {:.3f}, acc: {:.2%}".format(summary["loss"], summary["acc"]))
Example #13
Source File: model.py    From class-balanced-loss with MIT License 5 votes vote down vote up 
def get_estimator(**kwargs):
  """Construct an estimator."""
  cfg = utils.Config(kwargs)

  if cfg.tpu.get('name'):
    tf.logging.info('Using cluster resolver.')
    cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
        cfg.tpu.name, zone=cfg.tpu.zone, project=cfg.tpu.gcp_project)
    master = None
  else:
    cluster_resolver = None
    master = cfg.master

  tf.logging.info('Config:\n %s' % cfg)
  if cfg.tpu.enable:
    if not cfg.steps_per_epoch:
      raise ValueError('steps_per_epoch must be nonzero on TPU.')
    exp = tf.contrib.tpu.TPUEstimator(
        model_fn=model_fn,
        config=tf.contrib.tpu.RunConfig(
            cluster=cluster_resolver,
            master=master,
            model_dir=cfg.model_dir,
            tpu_config=tf.contrib.tpu.TPUConfig(
                iterations_per_loop=cfg.steps_per_epoch)),
        use_tpu=True,
        eval_on_tpu=False,
        # TPU requires these args, but they are ignored inside the input
        # function, which directly get train_batch_size or eval_batch_size.
        train_batch_size=cfg.dataset.train_batch_size,
        eval_batch_size=cfg.dataset.eval_batch_size,
        params=cfg,
    )
  else:
    exp = tf.estimator.Estimator(
        model_fn=model_fn, model_dir=cfg.model_dir, params=cfg)

  return exp