Python torch.optim() Examples
The following are 30
code examples of torch.optim().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch
, or try the search function
.
Example #1
Source File: train.py From pytorch-multigpu with MIT License | 7 votes |
def main(): best_acc = 0 device = 'cuda' if torch.cuda.is_available() else 'cpu' print('==> Preparing data..') transforms_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]) dataset_train = CIFAR10(root='../data', train=True, download=True, transform=transforms_train) train_loader = DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True, num_workers=args.num_worker) # there are 10 classes so the dataset name is cifar-10 classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') print('==> Making model..') net = pyramidnet() net = nn.DataParallel(net) net = net.to(device) num_params = sum(p.numel() for p in net.parameters() if p.requires_grad) print('The number of parameters of model is', num_params) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=args.lr) # optimizer = optim.SGD(net.parameters(), lr=args.lr, # momentum=0.9, weight_decay=1e-4) train(net, criterion, optimizer, train_loader, device)
Example #2
Source File: base_agent.py From 2D-Motion-Retargeting with MIT License | 7 votes |
def __init__(self, config, net): self.log_dir = config.log_dir self.model_dir = config.model_dir self.net = net self.clock = TrainClock() self.device = config.device self.use_triplet = config.use_triplet self.use_footvel_loss = config.use_footvel_loss # set loss function self.mse = nn.MSELoss() self.tripletloss = nn.TripletMarginLoss(margin=config.triplet_margin) self.triplet_weight = config.triplet_weight self.foot_idx = config.foot_idx self.footvel_loss_weight = config.footvel_loss_weight # set optimizer self.optimizer = optim.Adam(self.net.parameters(), config.lr) self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, 0.99)
Example #3
Source File: train_val.py From Collaborative-Learning-for-Weakly-Supervised-Object-Detection with MIT License | 6 votes |
def construct_graph(self): # Set the random seed torch.manual_seed(cfg.RNG_SEED) # Build the main computation graph self.net.create_architecture(self.imdb.num_classes, tag='default') # Define the loss # loss = layers['total_loss'] # Set learning rate and momentum lr = cfg.TRAIN.LEARNING_RATE params = [] for key, value in dict(self.net.named_parameters()).items(): if value.requires_grad: if 'bias' in key: params += [{'params':[value],'lr':lr*(cfg.TRAIN.DOUBLE_BIAS + 1), 'weight_decay': cfg.TRAIN.BIAS_DECAY and cfg.TRAIN.WEIGHT_DECAY or 0}] else: params += [{'params':[value],'lr':lr, 'weight_decay': cfg.TRAIN.WEIGHT_DECAY}] self.optimizer = torch.optim.SGD(params, momentum=cfg.TRAIN.MOMENTUM) # Write the train and validation information to tensorboard self.writer = tb.writer.FileWriter(self.tbdir) # self.valwriter = tb.writer.FileWriter(self.tbvaldir) return lr, self.optimizer
Example #4
Source File: torch_utils.py From pytorch-trpo with MIT License | 6 votes |
def fit(self, observations, labels): def closure(): predicted = self.predict(observations) loss = self.loss_fn(predicted, labels) self.optimizer.zero_grad() loss.backward() return loss old_params = parameters_to_vector(self.model.parameters()) for lr in self.lr * .5**np.arange(10): self.optimizer = optim.LBFGS(self.model.parameters(), lr=lr) self.optimizer.step(closure) current_params = parameters_to_vector(self.model.parameters()) if any(np.isnan(current_params.data.cpu().numpy())): print("LBFGS optimization diverged. Rolling back update...") vector_to_parameters(old_params, self.model.parameters()) else: return
Example #5
Source File: saver.py From L3C-PyTorch with GNU General Public License v3.0 | 6 votes |
def restore(self, modules, ckpt_p, strict=True, restore_restart=False): print('Restoring {}... (strict={})'.format(ckpt_p, strict)) map_location = None if pe.CUDA_AVAILABLE else 'cpu' state_dicts = torch.load(ckpt_p, map_location=map_location) # --- for key, m in modules.items(): # optim implements its own load_state_dict which does not have the `strict` keyword... if isinstance(m, optimizer.Optimizer): if restore_restart: print('Not restoring optimizer, --restore_restart given...') else: try: m.load_state_dict(state_dicts[key]) except ValueError as e: raise ValueError('Error while restoring Optimizer:', str(e)) else: try: m.load_state_dict(state_dicts[key], strict=strict) except RuntimeError as e: # loading error for n, module in sorted(m.named_modules()): print(n, module) raise e return self.get_itr_from_ckpt_p(ckpt_p)
Example #6
Source File: user.py From ConvLab with MIT License | 6 votes |
def imitating(self, epoch): """ train the user simulator by simple imitation learning (behavioral cloning) """ self.user.train() a_loss, t_loss = 0., 0. data_train_iter = batch_iter(self.data_train[0], self.data_train[1], self.data_train[2], self.data_train[3]) for i, data in enumerate(data_train_iter): self.optim.zero_grad() loss_a, loss_t = self.user_loop(data) a_loss += loss_a.item() t_loss += loss_t.item() loss = loss_a + loss_t loss.backward() self.optim.step() if (i+1) % self.print_per_batch == 0: a_loss /= self.print_per_batch t_loss /= self.print_per_batch logging.debug('<<user simulator>> epoch {}, iter {}, loss_a:{}, loss_t:{}'.format(epoch, i, a_loss, t_loss)) a_loss, t_loss = 0., 0. if (epoch+1) % self.save_per_epoch == 0: self.save(self.save_dir, epoch) self.user.eval()
Example #7
Source File: network.py From reconstructing_faces_from_voices with GNU General Public License v3.0 | 6 votes |
def get_network(net_type, params, train=True): net_params = params[net_type] net = net_params['network'](net_params['input_channel'], net_params['channels'], net_params['output_channel']) if params['GPU']: net.cuda() if train: net.train() optimizer = optim.Adam(net.parameters(), lr=params['lr'], betas=(params['beta1'], params['beta2'])) else: net.eval() net.load_state_dict(torch.load(net_params['model_path'])) optimizer = None return net, optimizer
Example #8
Source File: adversarial.py From Performance-RNN-PyTorch with MIT License | 6 votes |
def adversarial(args): train_adversarial(sess_path=args.session_path, batch_data_generator=args.batch_generator(args), model_load_path=args.generator_load_path, discriminator_load_path=args.discriminator_load_path, model_optimizer_class=getattr(optim, args.generator_optimizer), discriminator_optimizer_class=getattr(optim, args.discriminator_optimizer), model_gradient_clipping=args.generator_gradient_clipping, discriminator_gradient_clipping=args.discriminator_gradient_clipping, model_learning_rate=args.generator_learning_rate, discriminator_learning_rate=args.discriminator_learning_rate, reset_model_optimizer=args.reset_generator_optimizer, reset_discriminator_optimizer=args.reset_discriminator_optimizer, g_max_q_mean=args.g_max_q_mean, g_min_q_mean=args.g_min_q_mean, d_min_loss=args.d_min_loss, g_max_steps=args.g_max_steps, d_max_steps=args.d_max_steps, mc_sample_size=args.monte_carlo_sample_size, mc_sample_factor=args.monte_carlo_sample_factor, first_to_train=args.first_to_train, control_ratio=args.control_ratio, save_interval=args.save_interval, enable_logging=args.enable_logging)
Example #9
Source File: train_encodeDecode_pose.py From UnsupervisedGeometryAwareRepresentationLearning with GNU General Public License v3.0 | 6 votes |
def loadOptimizer(self, network, config_dict): params_all_id = list(map(id, network.parameters())) params_posenet_id = list(map(id, network.to_pose.parameters())) params_toOptimize = [p for p in network.parameters() if id(p) in params_posenet_id] params_static_id = [id_p for id_p in params_all_id if not id_p in params_posenet_id] # disable gradient computation for static params, saves memory and computation for p in network.parameters(): if id(p) in params_static_id: p.requires_grad = False print("Normal learning rate: {} params".format(len(params_posenet_id))) print("Static learning rate: {} params".format(len(params_static_id))) print("Total: {} params".format(len(params_all_id))) opt_params = [{'params': params_toOptimize, 'lr': config_dict['learning_rate']}] optimizer = torch.optim.Adam(opt_params, lr=config_dict['learning_rate']) #weight_decay=0.0005 return optimizer
Example #10
Source File: train.py From TreeGAN with MIT License | 6 votes |
def __init__(self, args): self.args = args # ------------------------------------------------Dataset---------------------------------------------- # self.data = BenchmarkDataset(root=args.dataset_path, npoints=args.point_num, uniform=True, class_choice=args.class_choice) self.dataLoader = torch.utils.data.DataLoader(self.data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4) print("Training Dataset : {} prepared.".format(len(self.data))) # ----------------------------------------------------------------------------------------------------- # # -------------------------------------------------Module---------------------------------------------- # self.G = Generator(batch_size=args.batch_size, features=args.G_FEAT, degrees=args.DEGREE, support=args.support).to(args.device) self.D = Discriminator(batch_size=args.batch_size, features=args.D_FEAT).to(args.device) self.optimizerG = optim.Adam(self.G.parameters(), lr=args.lr, betas=(0, 0.99)) self.optimizerD = optim.Adam(self.D.parameters(), lr=args.lr, betas=(0, 0.99)) self.GP = GradientPenalty(args.lambdaGP, gamma=1, device=args.device) print("Network prepared.") # ----------------------------------------------------------------------------------------------------- # # ---------------------------------------------Visualization------------------------------------------- # self.vis = visdom.Visdom(port=args.visdom_port) assert self.vis.check_connection() print("Visdom connected.") # ----------------------------------------------------------------------------------------------------- #
Example #11
Source File: imagenet.py From MobileNetV3-pytorch with MIT License | 6 votes |
def train_network(start_epoch, epochs, optim, model, train_loader, val_loader, criterion, mixup, device, dtype, batch_size, log_interval, csv_logger, save_path, claimed_acc1, claimed_acc5, best_test, local_rank, child): my_range = range if child else trange for epoch in my_range(start_epoch, epochs + 1): train_loss, train_accuracy1, train_accuracy5, = train(model, train_loader, mixup, epoch, optim, criterion, device, dtype, batch_size, log_interval, child) test_loss, test_accuracy1, test_accuracy5 = test(model, val_loader, criterion, device, dtype, child) optim.epoch_step() csv_logger.write({'epoch': epoch + 1, 'val_error1': 1 - test_accuracy1, 'val_error5': 1 - test_accuracy5, 'val_loss': test_loss, 'train_error1': 1 - train_accuracy1, 'train_error5': 1 - train_accuracy5, 'train_loss': train_loss}) save_checkpoint({'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_test, 'optimizer': optim.state_dict()}, test_accuracy1 > best_test, filepath=save_path, local_rank=local_rank) # TODO: save on the end of the cycle csv_logger.plot_progress(claimed_acc1=claimed_acc1, claimed_acc5=claimed_acc5) if test_accuracy1 > best_test: best_test = test_accuracy1 csv_logger.write_text('Best accuracy is {:.2f}% top-1'.format(best_test * 100.))
Example #12
Source File: net_util.py From ConvLab with MIT License | 6 votes |
def save_algorithm(algorithm, ckpt=None): '''Save all the nets for an algorithm''' agent = algorithm.agent net_names = algorithm.net_names model_prepath = agent.spec['meta']['model_prepath'] if ckpt is not None: model_prepath = f'{model_prepath}_ckpt-{ckpt}' for net_name in net_names: net = getattr(algorithm, net_name) model_path = f'{model_prepath}_{net_name}_model.pt' save(net, model_path) optim_name = net_name.replace('net', 'optim') optim = getattr(algorithm, optim_name, None) if optim is not None: # only trainable net has optim optim_path = f'{model_prepath}_{net_name}_optim.pt' save(optim, optim_path) logger.debug(f'Saved algorithm {util.get_class_name(algorithm)} nets {net_names} to {model_prepath}_*.pt')
Example #13
Source File: net_util.py From ConvLab with MIT License | 6 votes |
def load_algorithm(algorithm): '''Save all the nets for an algorithm''' agent = algorithm.agent net_names = algorithm.net_names if util.in_eval_lab_modes(): # load specific model in eval mode model_prepath = agent.spec['meta']['eval_model_prepath'] else: model_prepath = agent.spec['meta']['model_prepath'] logger.info(f'Loading algorithm {util.get_class_name(algorithm)} nets {net_names} from {model_prepath}_*.pt') for net_name in net_names: net = getattr(algorithm, net_name) model_path = f'{model_prepath}_{net_name}_model.pt' load(net, model_path) optim_name = net_name.replace('net', 'optim') optim = getattr(algorithm, optim_name, None) if optim is not None: # only trainable net has optim optim_path = f'{model_prepath}_{net_name}_optim.pt' load(optim, optim_path)
Example #14
Source File: SAC.py From Deep-reinforcement-learning-with-pytorch with MIT License | 6 votes |
def __init__(self): super(SAC, self).__init__() self.policy_net = Actor(state_dim).to(device) self.value_net = Critic(state_dim).to(device) self.Q_net = Q(state_dim, action_dim).to(device) self.Target_value_net = Critic(state_dim).to(device) self.replay_buffer = [Transition] * args.capacity self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) self.Q_optimizer = optim.Adam(self.Q_net.parameters(), lr=args.learning_rate) self.num_transition = 0 # pointer of replay buffer self.num_training = 1 self.writer = SummaryWriter('./exp-SAC') self.value_criterion = nn.MSELoss() self.Q_criterion = nn.MSELoss() for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): target_param.data.copy_(param.data) os.makedirs('./SAC_model/', exist_ok=True)
Example #15
Source File: main.py From ScenarioMeta with MIT License | 6 votes |
def get_optimizer(gradient_model, config): if config['support'].get('flexible_step', False): stop_parameters = list(filter(lambda p: p.requires_grad, gradient_model.stop_gate.parameters())) else: stop_parameters = [] init_parameters = list(filter(lambda p: p.requires_grad, gradient_model.model.parameters())) update_parameters = list(filter(lambda p: p.requires_grad, gradient_model.meta_lstms.parameters())) parameters = [ {'params': init_parameters, 'lr': config['lr']['init_lr']}, {'params': update_parameters, 'lr': config['lr']['update_lr']}, {'params': stop_parameters, 'lr': config['lr']['stop_lr']} ] optimizer = optim.Adam(parameters, **config['optim']) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.2, patience=2, verbose=True, min_lr=1e-6) return optimizer, scheduler
Example #16
Source File: DDPG.py From Deep-reinforcement-learning-with-pytorch with MIT License | 6 votes |
def __init__(self, state_dim, action_dim, max_action): self.actor = Actor(state_dim, action_dim, max_action).to(device) self.actor_target = Actor(state_dim, action_dim, max_action).to(device) self.actor_target.load_state_dict(self.actor.state_dict()) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=1e-4) self.critic = Critic(state_dim, action_dim).to(device) self.critic_target = Critic(state_dim, action_dim).to(device) self.critic_target.load_state_dict(self.critic.state_dict()) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=1e-3) self.replay_buffer = Replay_buffer() self.writer = SummaryWriter(directory) self.num_critic_update_iteration = 0 self.num_actor_update_iteration = 0 self.num_training = 0
Example #17
Source File: utils.py From PyTorchConv3D with Apache License 2.0 | 5 votes |
def get_optimizer(config, params): if config.optimizer == 'SGD': return torch.optim.SGD(params, config.learning_rate, config.momentum, weight_decay=config.weight_decay) elif config.optimizer == 'rmsprop': return torch.optim.RMSprop(params, config.learning_rate, weight_decay=config.weight_decay) elif config.optimizer == 'adam': return torch.optim.Adam(params, config.learning_rate, weight_decay=config.weight_decay) raise ValueError('Chosen optimizer is not supported, please choose from (SGD | adam | rmsprop)')
Example #18
Source File: test_agent.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(SAC, self).__init__() self.policy_net = Actor(state_dim).to(device) self.value_net = Critic(state_dim).to(device) self.Target_value_net = Critic(state_dim).to(device) self.Q_net1 = Q(state_dim, action_dim).to(device) self.Q_net2 = Q(state_dim, action_dim).to(device) self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) self.Q1_optimizer = optim.Adam(self.Q_net1.parameters(), lr=args.learning_rate) self.Q2_optimizer = optim.Adam(self.Q_net2.parameters(), lr=args.learning_rate) self.replay_buffer = [Transition] * args.capacity self.num_transition = 0 # pointer of replay buffer self.num_training = 0 self.writer = SummaryWriter('./test_agent') self.value_criterion = nn.MSELoss() self.Q1_criterion = nn.MSELoss() self.Q2_criterion = nn.MSELoss() for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): target_param.data.copy_(param.data) os.makedirs('./SAC_model/', exist_ok=True)
Example #19
Source File: PPO_MountainCar-v0.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(PPO, self).__init__() self.actor_net = Actor() self.critic_net = Critic() self.buffer = [] self.counter = 0 self.training_step = 0 self.writer = SummaryWriter('../exp') self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 3e-3) if not os.path.exists('../param'): os.makedirs('../param/net_param') os.makedirs('../param/img')
Example #20
Source File: PPO_CartPole_v0.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(PPO, self).__init__() self.actor_net = Actor() self.critic_net = Critic() self.buffer = [] self.counter = 0 self.training_step = 0 self.writer = SummaryWriter('../exp') self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 3e-3) if not os.path.exists('../param'): os.makedirs('../param/net_param') os.makedirs('../param/img')
Example #21
Source File: DQN_MountainCar-v0.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(DQN, self).__init__() self.target_net, self.act_net = Net(), Net() self.memory = [None]*self.capacity self.optimizer = optim.Adam(self.act_net.parameters(), self.learning_rate) self.loss_func = nn.MSELoss() self.writer = SummaryWriter('./DQN/logs')
Example #22
Source File: PPO2.py From Deep-reinforcement-learning-with-pytorch with MIT License | 5 votes |
def __init__(self): super(PPO, self).__init__() self.actor_net = Actor().float() self.critic_net = Critic().float() self.buffer = [] self.counter = 0 self.training_step = 0 self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 4e-3) if not os.path.exists('../param'): os.makedirs('../param/net_param') os.makedirs('../param/img')
Example #23
Source File: train_panoptic.py From seamseg with BSD 3-Clause "New" or "Revised" License | 5 votes |
def make_optimizer(config, model, epoch_length): body_config = config["body"] opt_config = config["optimizer"] sch_config = config["scheduler"] # Gather parameters from the network norm_parameters = [] other_parameters = [] for m in model.modules(): if any(isinstance(m, layer) for layer in NORM_LAYERS): norm_parameters += [p for p in m.parameters() if p.requires_grad] elif any(isinstance(m, layer) for layer in OTHER_LAYERS): other_parameters += [p for p in m.parameters() if p.requires_grad] assert len(norm_parameters) + len(other_parameters) == len([p for p in model.parameters() if p.requires_grad]), \ "Not all parameters that require grad are accounted for in the optimizer" # Set-up optimizer hyper-parameters parameters = [ { "params": norm_parameters, "lr": opt_config.getfloat("lr") if not body_config.getboolean("bn_frozen") else 0., "weight_decay": opt_config.getfloat("weight_decay") if opt_config.getboolean("weight_decay_norm") else 0. }, { "params": other_parameters, "lr": opt_config.getfloat("lr"), "weight_decay": opt_config.getfloat("weight_decay") } ] optimizer = optim.SGD( parameters, momentum=opt_config.getfloat("momentum"), nesterov=opt_config.getboolean("nesterov")) scheduler = scheduler_from_config(sch_config, optimizer, epoch_length) assert sch_config["update_mode"] in ("batch", "epoch") batch_update = sch_config["update_mode"] == "batch" total_epochs = sch_config.getint("epochs") return optimizer, scheduler, batch_update, total_epochs
Example #24
Source File: main_pytorch.py From deep_architect with MIT License | 5 votes |
def evaluate(self, inputs, outputs): network = hpt.PyTorchModel( inputs, outputs, {'in': torch.zeros(self.batch_size, self.in_features)}) optimizer = optim.Adam(network.parameters(), lr=self.learning_rate) network.train() for epoch in range(self.num_training_epochs): for _ in range(self.num_train_batches): data, target = self.train_dataset.next_batch(self.batch_size) optimizer.zero_grad() output = network({'in': torch.FloatTensor(data)}) loss = F.cross_entropy(output["out"], torch.LongTensor(target)) loss.backward() optimizer.step() if self.log_output_to_terminal and epoch % self.display_step == 0: print('epoch: %d' % (epoch + 1), 'train loss: %.6f' % loss.item()) # compute validation accuracy network.eval() correct = 0 with torch.no_grad(): for _ in range(self.num_val_batches): data, target = self.val_dataset.next_batch(self.batch_size) output = network({'in': torch.FloatTensor(data)}) pred = output["out"].data.max(1)[1] correct += pred.eq(torch.LongTensor(target).data).sum().item() val_acc = float(correct) / self.val_dataset.get_num_examples() print("validation accuracy: %0.4f" % val_acc) return {'validation_accuracy': val_acc}
Example #25
Source File: openchem_optimizer.py From OpenChem with MIT License | 5 votes |
def optimizer(self): if not isinstance(self._optimizer, torch.optim.Optimizer): raise ValueError('_optimizer must be an instance of ' 'torch.optim.Optimizer') return self._optimizer
Example #26
Source File: LSTM.py From Action-Recognition with MIT License | 5 votes |
def train(model, num_epoch, num_iter, lr=1e-3,rec_interval=2, disp_interval=10): optimizer = optim.Adam(model.parameters(), lr) loss_values = [] rec_step = 0 for eph in range(num_epoch): print('epoch {} starting ...'.format(eph)) avg_loss = 0 n_samples = 0 randpermed = torch.randperm(trainingData.size()[0])[:num_iter] for i in range(num_iter): model.hidden = (model.hidden[0].detach(), model.hidden[1].detach()) model.zero_grad() j = randpermed[i] X,Y = trainingData[j,:,:,:].view(300,1,75),labels[j,:] #print(X.size()) n_samples += len(X) X = autograd.Variable(X) #print(X) Y = autograd.Variable(Y.view(1)) y_hat = model(X) loss = F.cross_entropy(y_hat, Y) avg_loss += loss.data[0] if i % disp_interval == 0: print('epoch: %d iterations: %d loss :%g' % (eph, i, loss.data[0])) if rec_step%rec_interval==0: loss_values.append(loss.data[0]) loss.backward() optimizer.step() rec_step += 1 avg_loss /= n_samples #evaluating model accuracy #TrainAcc() print('epoch: {} <====train track===> avg_loss: {} \n'.format(eph, avg_loss)) return loss_values #l = train(model0, 10, 100, 2, 20)
Example #27
Source File: utils.py From whynot with MIT License | 5 votes |
def __init__(self, env, gamma=0.99, learning_rate=1e-3): # Discount factor gamma. self.gamma = gamma self.actor = NNPolicy(env) self.optimizer = optim.Adam(self.actor.parameters(), lr=learning_rate)
Example #28
Source File: ssds_train.py From ssds.pytorch with MIT License | 5 votes |
def configure_optimizer(self, trainable_param, cfg): if cfg.OPTIMIZER == 'sgd': optimizer = optim.SGD(trainable_param, lr=cfg.LEARNING_RATE, momentum=cfg.MOMENTUM, weight_decay=cfg.WEIGHT_DECAY) elif cfg.OPTIMIZER == 'rmsprop': optimizer = optim.RMSprop(trainable_param, lr=cfg.LEARNING_RATE, momentum=cfg.MOMENTUM, alpha=cfg.MOMENTUM_2, eps=cfg.EPS, weight_decay=cfg.WEIGHT_DECAY) elif cfg.OPTIMIZER == 'adam': optimizer = optim.Adam(trainable_param, lr=cfg.LEARNING_RATE, betas=(cfg.MOMENTUM, cfg.MOMENTUM_2), eps=cfg.EPS, weight_decay=cfg.WEIGHT_DECAY) else: AssertionError('optimizer can not be recognized.') return optimizer
Example #29
Source File: run.py From MobileNetV3-pytorch with MIT License | 5 votes |
def train(model, loader, mixup, epoch, optim, criterion, device, dtype, batch_size, log_interval, child): model.train() correct1, correct5 = 0, 0 enum_load = enumerate(loader) if child else enumerate(tqdm(loader)) for batch_idx, (data, t) in enum_load: data, t = data.to(device=device, dtype=dtype), t.to(device=device) data, target = mixup(data, t) optim.zero_grad() output = model(data) loss = criterion(output, target) loss.backward() optim.batch_step() corr = correct(output, t, topk=(1, 5)) correct1 += corr[0] correct5 += corr[1] if batch_idx % log_interval == 0 and not child: tqdm.write( 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}. ' 'Top-1 accuracy: {:.2f}%({:.2f}%). ' 'Top-5 accuracy: {:.2f}%({:.2f}%).'.format(epoch, batch_idx, len(loader), 100. * batch_idx / len(loader), loss.item(), 100. * corr[0] / batch_size, 100. * correct1 / (batch_size * (batch_idx + 1)), 100. * corr[1] / batch_size, 100. * correct5 / (batch_size * (batch_idx + 1)))) return loss.item(), correct1 / len(loader.sampler), correct5 / len(loader.sampler)
Example #30
Source File: imagenet.py From MobileNetV3-pytorch with MIT License | 5 votes |
def init_optimizer_and_mixup(args, train_loader, model, optim_state_dict=None): optimizer_class = torch.optim.SGD optimizer_params = {"lr": args.learning_rate, "momentum": args.momentum, "weight_decay": args.decay, "nesterov": True} if args.sched == 'clr': scheduler_class = CyclicLR scheduler_params = {"base_lr": args.min_lr, "max_lr": args.max_lr, "step_size_up": args.epochs_per_step * len(train_loader), "mode": args.mode, "last_epoch": args.start_step - 1} elif args.sched == 'multistep': scheduler_class = MultiStepLR scheduler_params = {"milestones": args.schedule, "gamma": args.gamma, "last_epoch": args.start_epoch - 1} elif args.sched == 'cosine': scheduler_class = CosineLR scheduler_params = {"max_epochs": args.epochs, "warmup_epochs": args.warmup, "iter_in_epoch": len(train_loader), "last_epoch": args.start_step - 1} elif args.sched == 'gamma': scheduler_class = StepLR scheduler_params = {"step_size": 30, "gamma": args.gamma, "last_epoch": args.start_epoch - 1} else: raise ValueError('Wrong scheduler!') optim = OptimizerWrapper(model, optimizer_class=optimizer_class, optimizer_params=optimizer_params, optimizer_state_dict=optim_state_dict, scheduler_class=scheduler_class, scheduler_params=scheduler_params, use_shadow_weights=args.dtype == torch.float16) mixup_start = len(train_loader) * args.mixup_warmup mixup_nr = len(train_loader) * (args.epochs - args.mixup_warmup) mixup = MixupScheduled(start_gamma=0, stop_gamma=args.mixup, wait_steps=mixup_start, nr_steps=mixup_nr, start_step=args.start_step, num_classes=args.num_classes, smooth_eps=args.smooth_eps) return optim, mixup