Python model.Model() Examples

def main(cache_dir):
    files_list = list(os.listdir(cache_dir))
    for file in files_list:
        full_filename = os.path.join(cache_dir, file)
        if os.path.isfile(full_filename):
            print("Processing {}".format(full_filename))
            m, stored_kwargs = pickle.load(open(full_filename, 'rb'))
            updated_kwargs = util.get_compatible_kwargs(model.Model, stored_kwargs)

            model_hash = util.object_hash(updated_kwargs)
            print("New hash -> " + model_hash)
            model_filename = os.path.join(cache_dir, "model_{}.p".format(model_hash))
            sys.setrecursionlimit(100000)
            pickle.dump((m,updated_kwargs), open(model_filename,'wb'), protocol=pickle.HIGHEST_PROTOCOL)

            os.remove(full_filename) 

def _eval(path_to_checkpoint, backbone_name, path_to_results_dir):
    dataset = AVA_video(EvalConfig.VAL_DATA)
    evaluator = Evaluator(dataset, path_to_results_dir)

    Log.i('Found {:d} samples'.format(len(dataset)))

    backbone = BackboneBase.from_name(backbone_name)()
    model = Model(backbone, dataset.num_classes(), pooler_mode=Config.POOLER_MODE,
                  anchor_ratios=Config.ANCHOR_RATIOS, anchor_sizes=Config.ANCHOR_SIZES,
                  rpn_pre_nms_top_n=TrainConfig.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=TrainConfig.RPN_POST_NMS_TOP_N).cuda()
    model.load(path_to_checkpoint)
    print("load from:",path_to_checkpoint)
    Log.i('Start evaluating with 1 GPU (1 batch per GPU)')
    mean_ap, detail = evaluator.evaluate(model)
    Log.i('Done')
    Log.i('mean AP = {:.4f}'.format(mean_ap))
    Log.i('\n' + detail) 

def sample(args):
    with open(os.path.join(args.save_dir, 'config.pkl'), 'rb') as f:
        saved_args = cPickle.load(f)
    with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'rb') as f:
        chars, vocab = cPickle.load(f)
    #Use most frequent char if no prime is given
    if args.prime == '':
        args.prime = chars[0]
    model = Model(saved_args, training=False)
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.global_variables())
        ckpt = tf.train.get_checkpoint_state(args.save_dir)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
            data = model.sample(sess, chars, vocab, args.n, args.prime,
                               args.sample).encode('utf-8')
            print(data.decode("utf-8")) 

def main():
    parser = argparse.ArgumentParser(description='test', formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        '-a', '--attributes',
        nargs='+',
        type=str,
        help='Specify attribute name for training. \nAll attributes can be found in list_attr_celeba.txt'
    )
    parser.add_argument(
        '-g', '--gpu',
        default='0',
        type=str,
        help='Specify GPU id. \ndefault: %(default)s. \nUse comma to seperate several ids, for example: 0,1'
    )
    args = parser.parse_args()

    celebA = Dataset(args.attributes)
    DNA_GAN = Model(args.attributes, is_train=True)
    run(config, celebA, DNA_GAN, gpu=args.gpu) 

def main(_):
    dataset = get_record_dataset(FLAGS.record_path)
    data_provider = slim.dataset_data_provider.DatasetDataProvider(dataset)
    image, label = data_provider.get(['image', 'label'])
    inputs, labels = tf.train.batch([image, label],
                                    batch_size=64,
                                    allow_smaller_final_batch=True)
    
    cls_model = model.Model(is_training=True)
    preprocessed_inputs = cls_model.preprocess(inputs)
    prediction_dict = cls_model.predict(preprocessed_inputs)
    loss_dict = cls_model.loss(prediction_dict, labels)
    loss = loss_dict['loss']
    postprocessed_dict = cls_model.postprocess(prediction_dict)
    acc = cls_model.accuracy(postprocessed_dict, labels)
    tf.summary.scalar('loss', loss)
    tf.summary.scalar('accuracy', acc)
    
    optimizer = tf.train.MomentumOptimizer(learning_rate=0.01, momentum=0.9)
    train_op = slim.learning.create_train_op(loss, optimizer,
                                             summarize_gradients=True)
    
    slim.learning.train(train_op=train_op, logdir=FLAGS.logdir,
                        save_summaries_secs=20, save_interval_secs=120) 

def main(_):
    
    with tf.device(FLAGS.device):
	
	model_save_path = 'model/'+FLAGS.model_save_path	
	# create directory if it does not exist
	if not tf.gfile.Exists(model_save_path):
		tf.gfile.MakeDirs(model_save_path)
	log_dir = 'logs/'+ model_save_path
	
	model = Model(learning_rate=0.0003, mode=FLAGS.mode)
	solver = Solver(model, model_save_path=model_save_path, log_dir=log_dir)
	
	# create directory if it does not exist
	if not tf.gfile.Exists(model_save_path):
		tf.gfile.MakeDirs(model_save_path)
	
	if FLAGS.mode == 'train':
		solver.train()
	elif FLAGS.mode == 'test':
		solver.test(checkpoint=FLAGS.checkpoint)
	else:
	    print 'Unrecognized mode.' 

def _eval(path_to_checkpoint: str, dataset_name: str, backbone_name: str, path_to_data_dir: str, path_to_results_dir: str):
    dataset = DatasetBase.from_name(dataset_name)(path_to_data_dir, DatasetBase.Mode.EVAL, Config.IMAGE_MIN_SIDE, Config.IMAGE_MAX_SIDE)
    evaluator = Evaluator(dataset, path_to_data_dir, path_to_results_dir)

    Log.i('Found {:d} samples'.format(len(dataset)))

    backbone = BackboneBase.from_name(backbone_name)(pretrained=False)
    model = Model(backbone, dataset.num_classes(), pooler_mode=Config.POOLER_MODE,
                  anchor_ratios=Config.ANCHOR_RATIOS, anchor_sizes=Config.ANCHOR_SIZES,
                  rpn_pre_nms_top_n=Config.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=Config.RPN_POST_NMS_TOP_N).cuda()
    model.load(path_to_checkpoint)

    Log.i('Start evaluating with 1 GPU (1 batch per GPU)')
    mean_ap, detail = evaluator.evaluate(model)
    Log.i('Done')

    Log.i('mean AP = {:.4f}'.format(mean_ap))
    Log.i('\n' + detail) 

def inference(ckpt, inference_input_file, inference_output_file, hparams, num_workers=1, jobid=0, scope=None, single_cell_fn=None):

    """Perform translation."""
    if hparams.inference_indices:
        assert num_workers == 1

    if not hparams.attention:
        model_creator = nmt_model.Model
    elif hparams.attention_architecture == "standard":
        model_creator = attention_model.AttentionModel
    elif hparams.attention_architecture in ["gnmt", "gnmt_v2"]:
        model_creator = gnmt_model.GNMTModel
    else:
        raise ValueError("Unknown model architecture")
    infer_model = create_infer_model(model_creator, hparams, scope, single_cell_fn)

    if num_workers == 1:
        single_worker_inference(infer_model, ckpt, inference_input_file, inference_output_file, hparams)
    else:
        multi_worker_inference(infer_model, ckpt, inference_input_file, inference_output_file, hparams, num_workers=num_workers, jobid=jobid) 

def main(_):
    # Specify which gpu to be used
    os.environ["CUDA_VISIBLE_DEVICES"] = '1'
    
    cls_model = model.Model(is_training=False, num_classes=61)
    if FLAGS.input_shape:
        input_shape = [
            int(dim) if dim != -1 else None 
            for dim in FLAGS.input_shape.split(',')
        ]
    else:
        input_shape = [None, None, None, 3]
    exporter.export_inference_graph(FLAGS.input_type,
                                    cls_model,
                                    FLAGS.trained_checkpoint_prefix,
                                    FLAGS.output_directory,
                                    input_shape) 

def main(_):
    
    with tf.device(FLAGS.device):
	
	model_save_path = 'model/'+FLAGS.model_save_path	
	# create directory if it does not exist
	if not tf.gfile.Exists(model_save_path):
		tf.gfile.MakeDirs(model_save_path)
	log_dir = 'logs/'+ model_save_path
	
	model = Model(learning_rate=0.0003, mode=FLAGS.mode)
	solver = Solver(model, model_save_path=model_save_path, log_dir=log_dir)
	
	# create directory if it does not exist
	if not tf.gfile.Exists(model_save_path):
		tf.gfile.MakeDirs(model_save_path)
	
	if FLAGS.mode == 'train':
		solver.train()
	elif FLAGS.mode == 'test':
		solver.test(checkpoint=FLAGS.checkpoint)
	else:
	    print 'Unrecognized mode.' 

def setup_train(self, model_file_path=None):
        self.model = Model(model_file_path)

        params = list(self.model.encoder.parameters()) + list(self.model.decoder.parameters()) + \
                 list(self.model.reduce_state.parameters())
        initial_lr = config.lr_coverage if config.is_coverage else config.lr
        if config.mode == 'MLE':
            self.optimizer = Adagrad(params, lr=0.15, initial_accumulator_value=0.1)
        else:
            self.optimizer = Adam(params, lr=initial_lr)

        start_iter, start_loss = 0, 0

        if model_file_path is not None:
            state = torch.load(model_file_path, map_location= lambda storage, location: storage)
            start_iter = state['iter']
            start_loss = state['current_loss']
        return start_iter, start_loss 

def train():
    model.train()
    total_loss = 0
    for word, char, label in tqdm(training_data, mininterval=1,
                                  desc='Train Processing', leave=False):

        optimizer.zero_grad()
        loss, _ = model(word, char, label)
        loss.backward()

        optimizer.step()
        optimizer.update_learning_rate()
        total_loss += loss.data
    return total_loss / training_data._stop_step


# ##############################################################################
# Save Model
# ############################################################################## 

def __init__(self, model_source="translate_Pre-train.pt"):
        model_source = torch.load(
            model_source, map_location=lambda storage, loc: storage)

        self.src_dict = model_source["dict"]["src"]
        self.idx2word = {v: k for k, v in model_source["dict"]["tgt"].items()}
        self.args = args = model_source["settings"]

        args.use_cuda = False
        model = Model(args)
        model.load_state_dict(model_source['model'])
        model = model.cpu()
        self.model = model.eval() 

def __init__(self, model_file_path):
        self.vocab = Vocab(config.vocab_path, config.vocab_size)
        self.batcher = Batcher(config.eval_data_path, self.vocab, mode='eval',
                               batch_size=config.batch_size, single_pass=True)
        self.model_file_path = model_file_path
        time.sleep(5)

        self.model = Model(model_file_path, is_eval=True) 

def init_model(self):
        self.model = model.Model() 

def initialize(self):
        self.logger = cfg.get_logger()
        self.info(repr(cfg))

        device = torch.device('cuda' if torch.cuda.is_available() \
                                         else 'cpu', index=0)
        device_num = -1 if device.type == 'cpu' else 0
        self.info(f'Use {device}')

        ########################################################################
        # Load data
        ########################################################################
        dev_upstream_file = config.RESULT_PATH / \
                        "doc_retri_bls/docretri.basic.nopageview/dev.jsonl"
        train_upstream_file = config.RESULT_PATH / \
                        "doc_retri_bls/docretri.basic.nopageview/train.jsonl"
        dev_corpus = DocIDCorpus(dev_upstream_file, train=False)
        train_corpus = DocIDCorpus(train_upstream_file, train=True)
        dev_corpus.initialize()
        train_corpus.initialize_from_exist_corpus(dev_corpus)

        ########################################################################
        # Build the model
        ########################################################################
        model = Model(weight=dev_corpus.weight_dict['glove.840B.300d'],
                      vocab_size=dev_corpus.vocab.get_vocab_size('tokens'),
                      embedding_dim=300, max_l=160, num_of_class=2)
        model.to(device)
        criterion = model.get_criterion()
        optimizer = optim.Adam(filter(lambda p: p.requires_grad,
                                      model.parameters()),
                               lr=cfg.start_lr)

        self.device_num = device_num
        self.train_corpus = train_corpus
        self.dev_corpus = dev_corpus
        self.model = model
        self.criterion = criterion
        self.optimizer = optimizer
        self.best_dev = -1 

def init_model(self):
        self.model = model.Model() 

def init_model(self):
        self.model = model.Model() 

def init_model(self):
        self.model = model.Model() 

def init_model(self):
        self.model = model.Model() 

def init_model(self):
        self.model = model.Model() 

def create_model(*args, **kwargs):
  ocr_model = model.Model(mparams=create_mparams(), *args, **kwargs)
  return ocr_model 

def create_model(self, charset=None):
    return model.Model(
        self.num_char_classes, self.seq_length, num_views=4, null_code=62,
        charset=charset) 

def demo(config):
    with open(config.word_emb_file, "r") as fh:
        word_mat = np.array(json.load(fh), dtype=np.float32)
    with open(config.char_emb_file, "r") as fh:
        char_mat = np.array(json.load(fh), dtype=np.float32)
    with open(config.test_meta, "r") as fh:
        meta = json.load(fh)

    model = Model(config, None, word_mat, char_mat, trainable=False, demo = True)
    demo = Demo(model, config) 

def _saveSession(self, sess):
        """ Save the model parameters and the variables
        Args:
            sess: the current session
        """
        tqdm.write('Checkpoint reached: saving model (don\'t stop the run)...')
        self.saveModelParams()
        model_name = self._getModelName()
        with open(model_name, 'w') as f:  # HACK: Simulate the old model existance to avoid rewriting the file parser
            f.write('This file is used internally by DeepQA to check the model existance. Please do not remove.\n')
        self.saver.save(sess, model_name)  # TODO: Put a limit size (ex: 3GB for the modelDir)
        tqdm.write('Model saved.') 

def __init__(self):
        """
        """
        # Model/dataset parameters
        self.args = None

        # Task specific object
        self.textData = None  # Dataset
        self.model = None  # Sequence to sequence model

        # Tensorflow utilities for convenience saving/logging
        self.writer = None
        self.saver = None
        self.modelDir = ''  # Where the model is saved
        self.globStep = 0  # Represent the number of iteration for the current model

        # TensorFlow main session (we keep track for the daemon)
        self.sess = None

        # Filename and directories constants
        self.MODEL_DIR_BASE = 'save/model'
        self.MODEL_NAME_BASE = 'model'
        self.MODEL_EXT = '.ckpt'
        self.CONFIG_FILENAME = 'params.ini'
        self.CONFIG_VERSION = '0.5'
        self.TEST_IN_NAME = 'data/test/samples.txt'
        self.TEST_OUT_SUFFIX = '_predictions.txt'
        self.SENTENCES_PREFIX = ['Q: ', 'A: '] 

def inference(ckpt,
              inference_input_file,
              inference_output_file,
              hparams,
              num_workers=1,
              jobid=0,
              scope=None):
  """Perform translation."""
  if hparams.inference_indices:
    assert num_workers == 1

  if not hparams.attention:
    model_creator = nmt_model.Model
  elif hparams.attention_architecture == "standard":
    model_creator = attention_model.AttentionModel
  elif hparams.attention_architecture in ["gnmt", "gnmt_v2"]:
    model_creator = gnmt_model.GNMTModel
  else:
    raise ValueError("Unknown model architecture")
  infer_model = model_helper.create_infer_model(model_creator, hparams, scope)

  if num_workers == 1:
    single_worker_inference(
        infer_model,
        ckpt,
        inference_input_file,
        inference_output_file,
        hparams)
  else:
    multi_worker_inference(
        infer_model,
        ckpt,
        inference_input_file,
        inference_output_file,
        hparams,
        num_workers=num_workers,
        jobid=jobid) 

def main(_):
    
    with tf.device(FLAGS.device):
	
	model_save_path = 'model/'+FLAGS.model_save_path	
	# create directory if it does not exist
	if not tf.gfile.Exists(model_save_path):
		tf.gfile.MakeDirs(model_save_path)
	log_dir = 'logs/'+ model_save_path
	
	if FLAGS.mode == 'test':
	    checkpoint = model_save_path+'/model'
	
	model = Model(learning_rate=0.0003, mode=FLAGS.mode)
	solver = Solver(model, model_save_path=model_save_path, 
					log_dir=log_dir, 
					training_size=int(FLAGS.training_size)
					)
	
	# create directory if it does not exist
	if not tf.gfile.Exists(model_save_path):
		tf.gfile.MakeDirs(model_save_path)
	
	if FLAGS.mode == 'train':
		solver.train()
	elif FLAGS.mode == 'test':
		solver.test(checkpoint=checkpoint)
	else:
	    print 'Unrecognized mode.' 

def get_model(self):
    cls = model.Model
    return cls(self.env_spec, self.global_step,
               target_network_lag=self.target_network_lag,
               sample_from=self.sample_from,
               get_policy=self.get_policy,
               get_baseline=self.get_baseline,
               get_objective=self.get_objective,
               get_trust_region_p_opt=self.get_trust_region_p_opt,
               get_value_opt=self.get_value_opt) 

def get_model(self):
    # vocab size is the number of distinct values that
    # could go into the memory key-value storage
    vocab_size = self.episode_width * self.batch_size
    return model.Model(
        self.input_dim, self.output_dim, self.rep_dim, self.memory_size,
        vocab_size, use_lsh=self.use_lsh)