Python torch.optim.Adam() Examples
The following are 30
code examples of torch.optim.Adam().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch.optim
, or try the search function
.
![](https://www.programcreek.com/common/static/images/search.png)
Example #1
Source File: optimizer.py From ACAN with MIT License | 7 votes |
def create_optimizer(args, optim_params): if args.optimizer == 'sgd': return optim.SGD(optim_params, args.lr, momentum=args.momentum, weight_decay=args.weight_decay) elif args.optimizer == 'adagrad': return optim.Adagrad(optim_params, args.lr, weight_decay=args.weight_decay) elif args.optimizer == 'adam': return optim.Adam(optim_params, args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer == 'amsgrad': return optim.Adam(optim_params, args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, amsgrad=True) elif args.optimizer == 'adabound': from adabound import AdaBound return AdaBound(optim_params, args.lr, betas=(args.beta1, args.beta2), final_lr=args.final_lr, gamma=args.gamma, weight_decay=args.weight_decay) else: assert args.optimizer == 'amsbound' from adabound import AdaBound return AdaBound(optim_params, args.lr, betas=(args.beta1, args.beta2), final_lr=args.final_lr, gamma=args.gamma, weight_decay=args.weight_decay, amsbound=True)
Example #2
Source File: base_agent.py From 2D-Motion-Retargeting with MIT License | 7 votes |
def __init__(self, config, net): self.log_dir = config.log_dir self.model_dir = config.model_dir self.net = net self.clock = TrainClock() self.device = config.device self.use_triplet = config.use_triplet self.use_footvel_loss = config.use_footvel_loss # set loss function self.mse = nn.MSELoss() self.tripletloss = nn.TripletMarginLoss(margin=config.triplet_margin) self.triplet_weight = config.triplet_weight self.foot_idx = config.foot_idx self.footvel_loss_weight = config.footvel_loss_weight # set optimizer self.optimizer = optim.Adam(self.net.parameters(), config.lr) self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, 0.99)
Example #3
Source File: train.py From pytorch-multigpu with MIT License | 7 votes |
def main(): best_acc = 0 device = 'cuda' if torch.cuda.is_available() else 'cpu' print('==> Preparing data..') transforms_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]) dataset_train = CIFAR10(root='../data', train=True, download=True, transform=transforms_train) train_loader = DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True, num_workers=args.num_worker) # there are 10 classes so the dataset name is cifar-10 classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') print('==> Making model..') net = pyramidnet() net = nn.DataParallel(net) net = net.to(device) num_params = sum(p.numel() for p in net.parameters() if p.requires_grad) print('The number of parameters of model is', num_params) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=args.lr) # optimizer = optim.SGD(net.parameters(), lr=args.lr, # momentum=0.9, weight_decay=1e-4) train(net, criterion, optimizer, train_loader, device)
Example #4
Source File: network.py From reconstructing_faces_from_voices with GNU General Public License v3.0 | 6 votes |
def get_network(net_type, params, train=True): net_params = params[net_type] net = net_params['network'](net_params['input_channel'], net_params['channels'], net_params['output_channel']) if params['GPU']: net.cuda() if train: net.train() optimizer = optim.Adam(net.parameters(), lr=params['lr'], betas=(params['beta1'], params['beta2'])) else: net.eval() net.load_state_dict(torch.load(net_params['model_path'])) optimizer = None return net, optimizer
Example #5
Source File: train.py From TreeGAN with MIT License | 6 votes |
def __init__(self, args): self.args = args # ------------------------------------------------Dataset---------------------------------------------- # self.data = BenchmarkDataset(root=args.dataset_path, npoints=args.point_num, uniform=True, class_choice=args.class_choice) self.dataLoader = torch.utils.data.DataLoader(self.data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4) print("Training Dataset : {} prepared.".format(len(self.data))) # ----------------------------------------------------------------------------------------------------- # # -------------------------------------------------Module---------------------------------------------- # self.G = Generator(batch_size=args.batch_size, features=args.G_FEAT, degrees=args.DEGREE, support=args.support).to(args.device) self.D = Discriminator(batch_size=args.batch_size, features=args.D_FEAT).to(args.device) self.optimizerG = optim.Adam(self.G.parameters(), lr=args.lr, betas=(0, 0.99)) self.optimizerD = optim.Adam(self.D.parameters(), lr=args.lr, betas=(0, 0.99)) self.GP = GradientPenalty(args.lambdaGP, gamma=1, device=args.device) print("Network prepared.") # ----------------------------------------------------------------------------------------------------- # # ---------------------------------------------Visualization------------------------------------------- # self.vis = visdom.Visdom(port=args.visdom_port) assert self.vis.check_connection() print("Visdom connected.") # ----------------------------------------------------------------------------------------------------- #
Example #6
Source File: TD3.py From Deep-reinforcement-learning-with-pytorch with MIT License | 6 votes |
def __init__(self, state_dim, action_dim, max_action): self.actor = Actor(state_dim, action_dim, max_action).to(device) self.actor_target = Actor(state_dim, action_dim, max_action).to(device) self.critic_1 = Critic(state_dim, action_dim).to(device) self.critic_1_target = Critic(state_dim, action_dim).to(device) self.critic_2 = Critic(state_dim, action_dim).to(device) self.critic_2_target = Critic(state_dim, action_dim).to(device) self.actor_optimizer = optim.Adam(self.actor.parameters()) self.critic_1_optimizer = optim.Adam(self.critic_1.parameters()) self.critic_2_optimizer = optim.Adam(self.critic_2.parameters()) self.actor_target.load_state_dict(self.actor.state_dict()) self.critic_1_target.load_state_dict(self.critic_1.state_dict()) self.critic_2_target.load_state_dict(self.critic_2.state_dict()) self.max_action = max_action self.memory = Replay_buffer(args.capacity) self.writer = SummaryWriter(directory) self.num_critic_update_iteration = 0 self.num_actor_update_iteration = 0 self.num_training = 0
Example #7
Source File: TD3_BipedalWalker-v2.py From Deep-reinforcement-learning-with-pytorch with MIT License | 6 votes |
def __init__(self, state_dim, action_dim, max_action): self.actor = Actor(state_dim, action_dim, max_action).to(device) self.actor_target = Actor(state_dim, action_dim, max_action).to(device) self.critic_1 = Critic(state_dim, action_dim).to(device) self.critic_1_target = Critic(state_dim, action_dim).to(device) self.critic_2 = Critic(state_dim, action_dim).to(device) self.critic_2_target = Critic(state_dim, action_dim).to(device) self.actor_optimizer = optim.Adam(self.actor.parameters()) self.critic_1_optimizer = optim.Adam(self.critic_1.parameters()) self.critic_2_optimizer = optim.Adam(self.critic_2.parameters()) self.actor_target.load_state_dict(self.actor.state_dict()) self.critic_1_target.load_state_dict(self.critic_1.state_dict()) self.critic_2_target.load_state_dict(self.critic_2.state_dict()) self.max_action = max_action self.memory = Replay_buffer(args.capacity) self.writer = SummaryWriter(directory) self.num_critic_update_iteration = 0 self.num_actor_update_iteration = 0 self.num_training = 0
Example #8
Source File: train.py From Performance-RNN-PyTorch with MIT License | 6 votes |
def load_session(): global sess_path, model_config, device, learning_rate, reset_optimizer try: sess = torch.load(sess_path) if 'model_config' in sess and sess['model_config'] != model_config: model_config = sess['model_config'] print('Use session config instead:') print(utils.dict2params(model_config)) model_state = sess['model_state'] optimizer_state = sess['model_optimizer_state'] print('Session is loaded from', sess_path) sess_loaded = True except: print('New session') sess_loaded = False model = PerformanceRNN(**model_config).to(device) optimizer = optim.Adam(model.parameters(), lr=learning_rate) if sess_loaded: model.load_state_dict(model_state) if not reset_optimizer: optimizer.load_state_dict(optimizer_state) return model, optimizer
Example #9
Source File: SAC.py From Deep-reinforcement-learning-with-pytorch with MIT License | 6 votes |
def __init__(self): super(SAC, self).__init__() self.policy_net = Actor(state_dim).to(device) self.value_net = Critic(state_dim).to(device) self.Q_net = Q(state_dim, action_dim).to(device) self.Target_value_net = Critic(state_dim).to(device) self.replay_buffer = [Transition] * args.capacity self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) self.Q_optimizer = optim.Adam(self.Q_net.parameters(), lr=args.learning_rate) self.num_transition = 0 # pointer of replay buffer self.num_training = 1 self.writer = SummaryWriter('./exp-SAC') self.value_criterion = nn.MSELoss() self.Q_criterion = nn.MSELoss() for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): target_param.data.copy_(param.data) os.makedirs('./SAC_model/', exist_ok=True)
Example #10
Source File: DDPG.py From Deep-reinforcement-learning-with-pytorch with MIT License | 6 votes |
def __init__(self, state_dim, action_dim, max_action): self.actor = Actor(state_dim, action_dim, max_action).to(device) self.actor_target = Actor(state_dim, action_dim, max_action).to(device) self.actor_target.load_state_dict(self.actor.state_dict()) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=1e-4) self.critic = Critic(state_dim, action_dim).to(device) self.critic_target = Critic(state_dim, action_dim).to(device) self.critic_target.load_state_dict(self.critic.state_dict()) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=1e-3) self.replay_buffer = Replay_buffer() self.writer = SummaryWriter(directory) self.num_critic_update_iteration = 0 self.num_actor_update_iteration = 0 self.num_training = 0
Example #11
Source File: agent_task.py From ConvLab with MIT License | 6 votes |
def __init__(self, model, corpus, args, name, tune_pi_only): self.model = model self.corpus = corpus self.args = args self.name = name self.raw_goal = None self.vec_goals_list = None self.logprobs = None print("Do we only tune the policy: {}".format(tune_pi_only)) self.opt = optim.SGD( [p for n, p in self.model.named_parameters() if 'c2z' in n or not tune_pi_only], lr=self.args.rl_lr, momentum=self.args.momentum, nesterov=(self.args.nesterov and self.args.momentum > 0)) # self.opt = optim.Adam(self.model.parameters(), lr=0.01) # self.opt = optim.RMSprop(self.model.parameters(), lr=0.0005) self.all_rewards = [] self.all_grads = [] self.model.train()
Example #12
Source File: run_mrc_ner.py From mrc-for-flat-nested-ner with Apache License 2.0 | 6 votes |
def load_model(config, num_train_steps, label_list): device = torch.device("cuda") n_gpu = torch.cuda.device_count() model = BertMRCNER(config, ) model.to(device) if n_gpu > 1: model = torch.nn.DataParallel(model) # prepare optimzier param_optimizer = list(model.named_parameters()) no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"] optimizer_grouped_parameters = [ {"params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], "weight_decay": 0.01}, {"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], "weight_decay": 0.0}] # optimizer = Adam(optimizer_grouped_parameters, lr=config.learning_rate) optimizer = BertAdam(optimizer_grouped_parameters, lr=config.learning_rate, warmup=config.warmup_proportion, t_total=num_train_steps, max_grad_norm=config.clip_grad) return model, optimizer, device, n_gpu
Example #13
Source File: utils.py From Attention-Gated-Networks with MIT License | 6 votes |
def get_optimizer(option, params): opt_alg = 'sgd' if not hasattr(option, 'optim') else option.optim if opt_alg == 'sgd': optimizer = optim.SGD(params, lr=option.lr_rate, momentum=0.9, nesterov=True, weight_decay=option.l2_reg_weight) if opt_alg == 'adam': optimizer = optim.Adam(params, lr=option.lr_rate, betas=(0.9, 0.999), weight_decay=option.l2_reg_weight) return optimizer
Example #14
Source File: main.py From ScenarioMeta with MIT License | 6 votes |
def get_optimizer(gradient_model, config): if config['support'].get('flexible_step', False): stop_parameters = list(filter(lambda p: p.requires_grad, gradient_model.stop_gate.parameters())) else: stop_parameters = [] init_parameters = list(filter(lambda p: p.requires_grad, gradient_model.model.parameters())) update_parameters = list(filter(lambda p: p.requires_grad, gradient_model.meta_lstms.parameters())) parameters = [ {'params': init_parameters, 'lr': config['lr']['init_lr']}, {'params': update_parameters, 'lr': config['lr']['update_lr']}, {'params': stop_parameters, 'lr': config['lr']['stop_lr']} ] optimizer = optim.Adam(parameters, **config['optim']) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.2, patience=2, verbose=True, min_lr=1e-6) return optimizer, scheduler
Example #15
Source File: SAC_dual_Q_net.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(SAC, self).__init__() self.policy_net = Actor(state_dim).to(device) self.value_net = Critic(state_dim).to(device) self.Target_value_net = Critic(state_dim).to(device) self.Q_net1 = Q(state_dim, action_dim).to(device) self.Q_net2 = Q(state_dim, action_dim).to(device) self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) self.Q1_optimizer = optim.Adam(self.Q_net1.parameters(), lr=args.learning_rate) self.Q2_optimizer = optim.Adam(self.Q_net2.parameters(), lr=args.learning_rate) self.replay_buffer = [Transition] * args.capacity self.num_transition = 0 # pointer of replay buffer self.num_training = 1 self.writer = SummaryWriter('./exp-SAC_dual_Q_network') self.value_criterion = nn.MSELoss() self.Q1_criterion = nn.MSELoss() self.Q2_criterion = nn.MSELoss() for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): target_param.data.copy_(param.data) os.makedirs('./SAC_model/', exist_ok=True)
Example #16
Source File: optimizers_lib.py From bgd with MIT License | 5 votes |
def adam(model, **kwargs): logger = kwargs.get("logger", None) assert(logger is not None) adam_params = { "eps": kwargs.get("eps", 1e-08), "lr": kwargs.get("lr", 0.001), "betas": kwargs.get("betas", (0.9, 0.999)), "weight_decay": kwargs.get("weight_decay", 0) } logger.info("ADAM params: " + str(adam_params)) all_params = [{'params': params, 'name': name, 'initial_lr': kwargs.get("lr", 0.001)} for l, (name, params) in enumerate(model.named_parameters())] return optim.Adam(all_params, **adam_params)
Example #17
Source File: PPO2.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(PPO, self).__init__() self.actor_net = Actor().float() self.critic_net = Critic().float() self.buffer = [] self.counter = 0 self.training_step = 0 self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 4e-3) if not os.path.exists('../param'): os.makedirs('../param/net_param') os.makedirs('../param/img')
Example #18
Source File: nscupa.py From Hierarchical-Sentiment with MIT License | 5 votes |
def main(args): print(32*"-"+"\nNeural Sentiment Classification with User & Product Attention:\n" + 32*"-") data_tl, (train_set, val_set, test_set), net, wdict = load(args) dataloader = DataLoader(data_tl.indexed_iter(train_set), batch_size=args.b_size, shuffle=True, num_workers=3, collate_fn=tuple_batch,pin_memory=True) dataloader_valid = DataLoader(data_tl.indexed_iter(val_set), batch_size=args.b_size, shuffle=False, num_workers=3, collate_fn=tuple_batch) dataloader_test = DataLoader(data_tl.indexed_iter(test_set), batch_size=args.b_size, shuffle=False, num_workers=3, collate_fn=tuple_batch,drop_last=True) criterion = torch.nn.CrossEntropyLoss() device = torch.device("cuda" if args.cuda else "cpu") if args.cuda: net.to(device) print("-"*20) optimizer = optim.Adam(net.parameters()) torch.nn.utils.clip_grad_norm(net.parameters(), args.clip_grad) for epoch in range(1, args.epochs + 1): print("\n-------EPOCH {}-------".format(epoch)) train(epoch,net,dataloader,device,msg="training",optimize=True,optimizer=optimizer,criterion=criterion) if args.snapshot: print("snapshot of model saved as {}".format(args.save+"_snapshot")) save(net,wdict,args.save+"_snapshot") train(epoch,net,dataloader_valid,device,msg="Validation") train(epoch,net,dataloader_test,device,msg="Evaluation") if args.save: print("model saved to {}".format(args.save)) save(net,wdict,args.save)
Example #19
Source File: test_agent.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(SAC, self).__init__() self.policy_net = Actor(state_dim).to(device) self.value_net = Critic(state_dim).to(device) self.Target_value_net = Critic(state_dim).to(device) self.Q_net1 = Q(state_dim, action_dim).to(device) self.Q_net2 = Q(state_dim, action_dim).to(device) self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) self.Q1_optimizer = optim.Adam(self.Q_net1.parameters(), lr=args.learning_rate) self.Q2_optimizer = optim.Adam(self.Q_net2.parameters(), lr=args.learning_rate) self.replay_buffer = [Transition] * args.capacity self.num_transition = 0 # pointer of replay buffer self.num_training = 0 self.writer = SummaryWriter('./test_agent') self.value_criterion = nn.MSELoss() self.Q1_criterion = nn.MSELoss() self.Q2_criterion = nn.MSELoss() for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): target_param.data.copy_(param.data) os.makedirs('./SAC_model/', exist_ok=True)
Example #20
Source File: PPO_pendulum.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): self.training_step = 0 self.anet = ActorNet().float() self.cnet = CriticNet().float() self.buffer = [] self.counter = 0 self.optimizer_a = optim.Adam(self.anet.parameters(), lr=1e-4) self.optimizer_c = optim.Adam(self.cnet.parameters(), lr=3e-4)
Example #21
Source File: han.py From Hierarchical-Sentiment with MIT License | 5 votes |
def main(args): print(32*"-"+"\nHierarchical Attention Network:\n" + 32*"-") data_tl, (train_set, val_set, test_set), net, wdict = load(args) dataloader = DataLoader(data_tl.indexed_iter(train_set), batch_size=args.b_size, shuffle=True, num_workers=3, collate_fn=tuple_batch,pin_memory=True) dataloader_valid = DataLoader(data_tl.indexed_iter(val_set), batch_size=args.b_size, shuffle=False, num_workers=3, collate_fn=tuple_batch) dataloader_test = DataLoader(data_tl.indexed_iter(test_set), batch_size=args.b_size, shuffle=False, num_workers=3, collate_fn=tuple_batch) criterion = torch.nn.CrossEntropyLoss() device = torch.device("cuda" if args.cuda else "cpu") if args.cuda: net.to(device) print("-"*20) optimizer = optim.Adam(net.parameters()) torch.nn.utils.clip_grad_norm(net.parameters(), args.clip_grad) for epoch in range(1, args.epochs + 1): print("\n-------EPOCH {}-------".format(epoch)) train(epoch,net,dataloader,device,msg="training",optimize=True,optimizer=optimizer,criterion=criterion) if args.snapshot: print("snapshot of model saved as {}".format(args.save+"_snapshot")) save(net,wdict,args.save+"_snapshot") train(epoch,net,dataloader_valid,device,msg="Validation") train(epoch,net,dataloader_test,device,msg="Evaluation") if args.save: print("model saved to {}".format(args.save)) save(net,wdict,args.save)
Example #22
Source File: wrapper.py From metalearn-leap with Apache License 2.0 | 5 votes |
def __init__(self, model, optimizer_cls, meta_optimizer_cls, optimizer_kwargs, meta_optimizer_kwargs, criterion): self.criterion = criterion self.model = model self.optimizer_cls = maml.SGD if optimizer_cls.lower() == 'sgd' else maml.Adam self.meta = maml.MAML(optimizer_cls=self.optimizer_cls, criterion=criterion, model=model, tensor=False, **optimizer_kwargs) self.meta_optimizer_cls = optim.SGD if meta_optimizer_cls.lower() == 'sgd' else optim.Adam self.optimizer_kwargs = optimizer_kwargs self.meta_optimizer = self.meta_optimizer_cls(self.meta.parameters(), **meta_optimizer_kwargs)
Example #23
Source File: LSTM.py From Action-Recognition with MIT License | 5 votes |
def train(model, num_epoch, num_iter, lr=1e-3,rec_interval=2, disp_interval=10): optimizer = optim.Adam(model.parameters(), lr) loss_values = [] rec_step = 0 for eph in range(num_epoch): print('epoch {} starting ...'.format(eph)) avg_loss = 0 n_samples = 0 randpermed = torch.randperm(trainingData.size()[0])[:num_iter] for i in range(num_iter): model.hidden = (model.hidden[0].detach(), model.hidden[1].detach()) model.zero_grad() j = randpermed[i] X,Y = trainingData[j,:,:,:].view(300,1,75),labels[j,:] #print(X.size()) n_samples += len(X) X = autograd.Variable(X) #print(X) Y = autograd.Variable(Y.view(1)) y_hat = model(X) loss = F.cross_entropy(y_hat, Y) avg_loss += loss.data[0] if i % disp_interval == 0: print('epoch: %d iterations: %d loss :%g' % (eph, i, loss.data[0])) if rec_step%rec_interval==0: loss_values.append(loss.data[0]) loss.backward() optimizer.step() rec_step += 1 avg_loss /= n_samples #evaluating model accuracy #TrainAcc() print('epoch: {} <====train track===> avg_loss: {} \n'.format(eph, avg_loss)) return loss_values #l = train(model0, 10, 100, 2, 20)
Example #24
Source File: basicLSTM.py From Action-Recognition with MIT License | 5 votes |
def train(model, num_epoch, batchSize = 5, lr=1e-3,rec_interval=50, disp_interval=25): global data, labels optimizer = optim.Adam(model.parameters(), lr) loss_values = [] rec_step = 0 for eph in range(num_epoch): print('epoch {} starting ...'.format(eph)) avg_loss = 0 n_samples = 0 num_iter = len(data)//batchSize randpermed = torch.randperm(len(data)) for i in range(num_iter): model.hidden = (model.hidden[0].detach(), model.hidden[1].detach()) model.zero_grad() for k in range(batchSize): j = randpermed[i*batchSize + k] X= data[j].view(data[j].size()[0],1,75) Y = torch.LongTensor(1) Y[0]=labels[j] #print(X.size()) n_samples += len(X) X = autograd.Variable(X) #print(X) Y = autograd.Variable(Y) y_hat = model(X) loss = F.cross_entropy(y_hat, Y) avg_loss += loss.data[0] if rec_step%rec_interval==0: loss_values.append(loss.data[0]) loss.backward(retain_graph = True) rec_step += 1 optimizer.step() if i % disp_interval == 0: print('epoch: %d iterations: %d loss :%g' % (eph, i, loss.data[0])) avg_loss /= n_samples #evaluating model accuracy #TrainAcc() print('epoch: {} <====train track===> avg_loss: {} \n'.format(eph, avg_loss)) return loss_values
Example #25
Source File: LSTM.py From Action-Recognition with MIT License | 5 votes |
def train(model, num_epoch, num_iter, lr=1e-3,rec_interval=2, disp_interval=10): optimizer = optim.Adam(model.parameters(), lr) loss_values = [] rec_step = 0 for eph in range(num_epoch): print('epoch {} starting ...'.format(eph)) avg_loss = 0 n_samples = 0 randpermed = torch.randperm(trainingData.size()[0])[:num_iter] for i in range(num_iter): model.hidden = (model.hidden[0].detach(), model.hidden[1].detach()) model.zero_grad() j = randpermed[i] X,Y = trainingData[j,:,:,:].view(300,1,75),labels[j,:] #print(X.size()) n_samples += len(X) X = autograd.Variable(X) #print(X) Y = autograd.Variable(Y.view(1)) y_hat = model(X) loss = F.cross_entropy(y_hat, Y) avg_loss += loss.data[0] if i % disp_interval == 0: print('epoch: %d iterations: %d loss :%g' % (eph, i, loss.data[0])) if rec_step%rec_interval==0: loss_values.append(loss.data[0]) loss.backward() optimizer.step() rec_step += 1 avg_loss /= n_samples #evaluating model accuracy #TrainAcc() print('epoch: {} <====train track===> avg_loss: {} \n'.format(eph, avg_loss)) return loss_values #l = train(model0, 10, 100, 2, 20)
Example #26
Source File: main_pytorch.py From deep_architect with MIT License | 5 votes |
def evaluate(self, inputs, outputs): network = hpt.PyTorchModel( inputs, outputs, {'in': torch.zeros(self.batch_size, self.in_features)}) optimizer = optim.Adam(network.parameters(), lr=self.learning_rate) network.train() for epoch in range(self.num_training_epochs): for _ in range(self.num_train_batches): data, target = self.train_dataset.next_batch(self.batch_size) optimizer.zero_grad() output = network({'in': torch.FloatTensor(data)}) loss = F.cross_entropy(output["out"], torch.LongTensor(target)) loss.backward() optimizer.step() if self.log_output_to_terminal and epoch % self.display_step == 0: print('epoch: %d' % (epoch + 1), 'train loss: %.6f' % loss.item()) # compute validation accuracy network.eval() correct = 0 with torch.no_grad(): for _ in range(self.num_val_batches): data, target = self.val_dataset.next_batch(self.batch_size) output = network({'in': torch.FloatTensor(data)}) pred = output["out"].data.max(1)[1] correct += pred.eq(torch.LongTensor(target).data).sum().item() val_acc = float(correct) / self.val_dataset.get_num_examples() print("validation accuracy: %0.4f" % val_acc) return {'validation_accuracy': val_acc}
Example #27
Source File: custom_lr_schedulers.py From nsf with MIT License | 5 votes |
def main(): model = nn.Linear(5, 3) optimizer = optim.Adam(model.parameters(), lr=1e-3) warm_up_steps = 0 total_steps = 100 scheduler = CosineAnnealingWarmUpLR(optimizer, warm_up_steps, total_steps) lrs = [] for _ in range(total_steps): optimizer.zero_grad() optimizer.step() scheduler.step() lrs.append(scheduler.get_lr()[0]) print(lrs)
Example #28
Source File: ssds_train.py From ssds.pytorch with MIT License | 5 votes |
def configure_optimizer(self, trainable_param, cfg): if cfg.OPTIMIZER == 'sgd': optimizer = optim.SGD(trainable_param, lr=cfg.LEARNING_RATE, momentum=cfg.MOMENTUM, weight_decay=cfg.WEIGHT_DECAY) elif cfg.OPTIMIZER == 'rmsprop': optimizer = optim.RMSprop(trainable_param, lr=cfg.LEARNING_RATE, momentum=cfg.MOMENTUM, alpha=cfg.MOMENTUM_2, eps=cfg.EPS, weight_decay=cfg.WEIGHT_DECAY) elif cfg.OPTIMIZER == 'adam': optimizer = optim.Adam(trainable_param, lr=cfg.LEARNING_RATE, betas=(cfg.MOMENTUM, cfg.MOMENTUM_2), eps=cfg.EPS, weight_decay=cfg.WEIGHT_DECAY) else: AssertionError('optimizer can not be recognized.') return optimizer
Example #29
Source File: wrapper.py From metalearn-leap with Apache License 2.0 | 5 votes |
def __init__(self, model, optimizer_cls, meta_optimizer_cls, optimizer_kwargs, meta_optimizer_kwargs, criterion): super(_FOWrapper, self).__init__(criterion, model, optimizer_cls, optimizer_kwargs) self.meta_optimizer_cls = optim.SGD if meta_optimizer_cls.lower() == 'sgd' else optim.Adam self.meta_optimizer_kwargs = meta_optimizer_kwargs self._counter = 0 self._updates = None self._original = clone_state_dict(self.model.state_dict(keep_vars=True)) params = [p for p in self._original.values() if getattr(p, 'requires_grad', False)] self.meta_optimizer = self.meta_optimizer_cls(params, **meta_optimizer_kwargs)
Example #30
Source File: registry.py From neural-pipeline with MIT License | 5 votes |
def _init_by_params(self, params: {}) -> optim.Adam: return optim.Adam(params=params['params'], lr=params['lr'] if 'lr' in params else 1e-3, betas=params['betas'] if 'betas' in params else (0.9, 0.999), eps=params['eps'] if 'eps' in params else 1e-8, weight_decay=params['weight_decay'] if 'weight_decay' in params else 0, amsgrad=params['amsgrad'] if 'amsgrad' in params else False)