Python chainer.cuda.get_device_from_id() Examples

def __call__(self, x):
        """Apply layer normalization to given input.
        Args:
            x (~chainer.Variable): Batch vectors.
                Shape of this value must be `(batch_size, unit_size)`,
                e.g., the output of :func:`~chainer.functions.linear`.
        Returns:
            ~chainer.Variable: Output of the layer normalization.
        """
#         if self.gamma.data is None:
#             self._initialize_params(x.size // x.shape[0])

        if self.has_uninitialized_params:
            with cuda.get_device_from_id(self._device_id):
                self._initialize_params(x.size // x.shape[0])

        return layer_normalization(
                x, self.gamma, self.beta, self.eps) 

def __call__(self, trainer):
        if self.force_enlarge_dataset:
            self.force_enlarge_dataset = False
            self.enlarge_dataset(trainer)

        if self.trigger(trainer):
            with cuda.get_device_from_id(trainer.updater.get_optimizer('main').target._device_id):
                loss = trainer.observation.get('fast_validation/main/loss', None)
                if loss is None:
                    return
                queue_data = loss.data if isinstance(loss, Variable) else loss
                self.queue.append(float(queue_data))
                if len(self.queue) >= self.maxlen:
                    if not self.training_converged():
                        return

                    self.enlarge_dataset(trainer) 

def __call__(self, trainer):
        iteration = trainer.updater.iteration

        with cuda.get_device_from_id(trainer.updater.get_optimizer('main').target._device_id), chainer.using_config('train', False):
            self.xp = np if trainer.updater.get_optimizer('main').target._device_id < 0 else cuda.cupy
            image = self.xp.asarray(self.image)
            predictor = trainer.updater.get_optimizer('main').target.predictor
            predictions, rois, bboxes = predictor(image[self.xp.newaxis, ...])

            backprop_visualizations = []
            for visanchor in self.visualization_anchors:
                vis_target = predictor
                for target in visanchor:
                    vis_target = getattr(vis_target, target)
                backprop_visualizations.append(self.visual_backprop.perform_visual_backprop(vis_target))

            self.render_rois(predictions, rois, bboxes, iteration, self.image.copy(), backprop_vis=backprop_visualizations) 

def __init__(self, model, optimizer,
                 minibatch_size=128,
                 states_per_epoch=2048,
                 action_wrapper='discrete',
                 entropy_coef=0.01, gpu=None):
        if gpu is not None and gpu >= 0:
            cuda.get_device_from_id(gpu).use()
            self.model.to_gpu(device=gpu)

        self.model = model
        self.optimizer = optimizer
        self.minibatch_size = minibatch_size
        self.states_per_epoch = states_per_epoch
        self.average_loss = 1e38
        self.action_wrapper = action_wrapper
        self.entropy_coef = entropy_coef
        self.xp = self.model.xp 

def __call__(self, c, h, x):
        """Returns new cell state and updated output of LSTM.

        Args:
            c (~chainer.Variable): Cell states of LSTM units.
            h (~chainer.Variable): Output at the previous time step.
            x (~chainer.Variable): A new batch from the input sequence.

        Returns:
            tuple of ~chainer.Variable: Returns ``(c_new, h_new)``, where
                ``c_new`` represents new cell state, and ``h_new`` is updated
                output of LSTM units.

        """
        if self.upward.has_uninitialized_params:
            in_size = x.size // x.shape[0]
            with cuda.get_device_from_id(self._device_id):
                self.upward._initialize_params(in_size)
                self._initialize_params()

        lstm_in = self.upward_ln(self.upward(x))
        if h is not None:
            lstm_in += self.lateral_ln(self.lateral(h))
        if c is None:
            xp = self.xp
            with cuda.get_device_from_id(self._device_id):
                c = variable.Variable(
                    xp.zeros((x.shape[0], self.state_size), dtype=x.dtype),
                    volatile='auto')
        c_next, ungated_h, o_gate = lstm_with_ungated_output(c, lstm_in)
        h = o_gate * self.output_ln(ungated_h)
        return c_next, h 

def generate():
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', '-g', type=int, default=-1)
    parser.add_argument('--gen', type=str, default=None)
    parser.add_argument('--depth', '-d', type=int, default=0)
    parser.add_argument('--out', '-o', type=str, default='img/')
    parser.add_argument('--num', '-n', type=int, default=10)
    args = parser.parse_args()
    
    gen = network.Generator(depth=args.depth)
    print('loading generator model from ' + args.gen)
    serializers.load_npz(args.gen, gen)

    if args.gpu >= 0:
        cuda.get_device_from_id(0).use()
        gen.to_gpu()

    xp = gen.xp
        
    z1 = gen.z(1)
    z2 = gen.z(1)

    for i in range(args.num):
        print(i)
        p = i / (args.num-1)
        z = z1 * p + z2 * (1 - p)
        x = gen(z, alpha=1.0)
        x = chainer.cuda.to_cpu(x.data)
        
        img = x[0].copy()
        filename = os.path.join(args.out, 'gen_%04d.png'%i)
        utils.save_image(img, filename) 

def __call__(self, trainer):
        if self.force_shift:
            self.shift_attribute(trainer)
            self.force_shift = False
            return

        if self.trigger(trainer):
            with cuda.get_device_from_id(trainer.updater.get_optimizer('main').target._device_id):
                loss = trainer.observation.get('validation/main/loss', None)
                if loss is None:
                    return
                queue_data = loss.data if isinstance(loss, Variable) else loss
                self.queue.append(float(queue_data))
                if len(self.queue) == self.queue.maxlen:
                    # check whether we need to shift attribute
                    deltas = []
                    rotated_queue = self.queue.copy()
                    rotated_queue.rotate(-1)
                    rotated_queue.pop()
                    for element_1, element_2 in zip(self.queue, rotated_queue):
                        deltas.append(abs(element_1 - element_2))

                    delta = sum(deltas) / len(deltas)
                    # if change over last 5 validations was lower than min change shift attribute
                    if delta < self.min_delta:
                        self.shift_attribute(trainer) 

def __call__(self, x, **kwargs):
        argument.check_unexpected_kwargs(
            kwargs, test='test argument is not supported anymore. '
            'Use chainer.using_config')
        finetune, = argument.parse_kwargs(kwargs, ('finetune', False))

        if hasattr(self, 'gamma'):
            gamma = self.gamma
        else:
            with cuda.get_device_from_id(self._device_id):
                gamma = variable.Variable(self.xp.ones(
                    self.avg_mean.shape, dtype=x.dtype))
        if hasattr(self, 'beta'):
            beta = self.beta
        else:
            with cuda.get_device_from_id(self._device_id):
                beta = variable.Variable(self.xp.zeros(
                    self.avg_mean.shape, dtype=x.dtype))
        if configuration.config.train:
            if finetune:
                self.N += 1
                decay = 1. - 1. / self.N
            else:
                decay = self.decay

            ret = batch_normalization(
                x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
                running_var=self.avg_var, decay=decay)
        else:
            # Use running average statistics or fine-tuned statistics.
            mean = variable.Variable(self.avg_mean)
            var = variable.Variable(self.avg_var)
            ret = fixed_batch_normalization(
                x, gamma, beta, mean, var, self.eps)
        return ret 

def generate():
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', '-g', type=int, default=-1)
    parser.add_argument('--model', '-m', type=str, default=None)
    parser.add_argument('--id', '-i', type=int, default=0)
    parser.add_argument('--inf', type=int, default=10)
    parser.add_argument('--outf', type=int, default=10)
    args = parser.parse_args()

    test = dataset.MovingMnistDataset(0, 10000, args.inf, args.outf)

    model = network.MovingMnistNetwork(sz=[128, 64, 64], n=2, directory="img/")

    if args.model != None:
        print( "loading model from " + args.model )
        serializers.load_npz(args.model, model)

    x, t = test[args.id]

    x = np.expand_dims(x, 0)
    t = np.expand_dims(t, 0)

    if args.gpu >= 0:
        cuda.get_device_from_id(0).use()
        model.to_gpu()
        x = cuda.cupy.array(x)
        t = cuda.cupy.array(t)

    res = model(Variable(x), Variable(t)) 

def __init__(self, model, optimizer, obs_normalizer=None,
                 update_interval=3072, minibatch_size=3072, epochs=1,
                 entropy_coef=1e-3, loss_decay=0.99, entropy_decay=0.99,
                 discriminator_value_offset=1e-8, noisy_label=False,
                 noisy_label_range=0.3, gpu=None):
        self.model = model
        self.optimizer = optimizer
        self.obs_normalizer = obs_normalizer

        if gpu is not None and gpu >= 0:
            cuda.get_device_from_id(gpu).use()
            self.model.to_gpu(device=gpu)
            if obs_normalizer is not None:
                self.obs_normalizer.to_gpu(device=gpu)
        self.xp = self.model.xp

        self.epochs = epochs
        self.update_interval = update_interval
        self.minibatch_size = minibatch_size
        self.epochs = epochs
        self.entropy_coef = entropy_coef
        self.loss_decay = loss_decay
        self.entropy_decay = entropy_decay
        self.average_loss = 0.
        self.accuracy_gen = 0.
        self.accuracy_exp = 0.
        self.average_entropy = 0.
        self.discriminator_value_offset = discriminator_value_offset
        self.noisy_label = noisy_label
        self.noisy_label_range = noisy_label_range
        self._reset_trajectories() 

def dump_token_embeddings(vocab_file, options_file, weight_file, outfile,
                          gpu=-1, batchsize=128):
    '''
    Given an input vocabulary file, dump all the token embeddings to the
    outfile.  The result can be used as the embedding_weight_file when
    constructing a BidirectionalLanguageModel.
    '''
    with open(options_file, 'r') as fin:
        options = json.load(fin)
    max_word_length = options['char_cnn']['max_characters_per_token']

    vocab = UnicodeCharsVocabulary(vocab_file, max_word_length)
    batcher = Batcher(vocab_file, max_word_length)

    model = Elmo(
        options_file,
        weight_file,
        num_output_representations=1,
        requires_grad=False,
        do_layer_norm=False,
        dropout=0.)

    tokens = [vocab.id_to_word(i) for i in range(vocab.size)]
    n_tokens = len(tokens)

    # (batch_size, timesteps, 50)
    if gpu >= 0:
        cuda.get_device_from_id(gpu).use()
        model.to_gpu()

    all_embeddings = []
    with chainer.using_config("train", False), \
            chainer.no_backprop_mode():
        for minibatch in minibatch_iterator(tqdm.tqdm(tokens, total=n_tokens),
                                            batchsize):
            char_ids = batcher.batch_sentences([minibatch], add_bos_eos=False)
            char_ids = model.xp.asarray(char_ids)  # to gpu
            embeddings = model._elmo_lstm._token_embedder\
                                         .forward(char_ids)['token_embedding']
            # (batch_size, sequence_length + 2, embedding_dim)
            embeddings = embeddings[:, 1:-1]  # del bos and eos
            embeddings = embeddings[0]
            embeddings = cuda.to_cpu(embeddings.array)
            all_embeddings.append(embeddings)

    all_embeddings = numpy.concatenate(all_embeddings, axis=0)
    with h5py.File(outfile, 'w') as fout:
        ds = fout.create_dataset(
            'embedding',
            all_embeddings.shape,
            dtype='float32',
            data=all_embeddings) 

def dump_bilm_embeddings(vocab_file, dataset_file, options_file,
                         weight_file, outfile, gpu=-1,
                         batchsize=32):
    with open(options_file, 'r') as fin:
        options = json.load(fin)
    max_word_length = options['char_cnn']['max_characters_per_token']

    vocab = UnicodeCharsVocabulary(vocab_file, max_word_length)
    batcher = Batcher(vocab_file, max_word_length)

    model = Elmo(
        options_file,
        weight_file,
        num_output_representations=1,
        requires_grad=False,
        do_layer_norm=False,
        dropout=0.)
    if gpu >= 0:
        cuda.get_device_from_id(gpu).use()
        model.to_gpu()

    # (batch_size, timesteps, 50)
    # TODO(sosk): preencoding token embedding for acceleration
    with chainer.using_config("train", False), \
            chainer.no_backprop_mode():
        sentence_id = 0
        n_lines = sum([1 for _ in open(dataset_file, 'r')])
        with open(dataset_file, 'r') as fin, h5py.File(outfile, 'w') as fout:
            for minibatch in minibatch_iterator(tqdm.tqdm(fin, total=n_lines),
                                                batchsize):
                sentences = [line.strip().split() for line in minibatch]
                char_ids = batcher.batch_sentences(
                    sentences, add_bos_eos=False)
                char_ids = model.xp.asarray(char_ids)
                mb_outs = model.forward(char_ids)
                mb_embedding_layers = mb_outs['elmo_layers']
                # [(batch_size, max_sequence_length, embedding_dim), ..., x n_layers]
                # Note that embedding layers have already trushed bos & eos
                # But they contains padding
                mb_mask = mb_outs['mask']
                mb_concat_embedding_layers = cuda.to_cpu(
                    model.xp.stack([mb_emb.array for mb_emb in mb_embedding_layers], axis=1))
                # (batch_size, n_layers=3, max_sequence_length, embedding_dim)
                for mask, concat_embedding_layers in zip(mb_mask, mb_concat_embedding_layers):
                    # remove pads
                    length = int(mask.sum())
                    concat_embedding_layers = concat_embedding_layers[:, :length]
                    # (n_layers=3, sequence_length, embedding_dim)
                    ds = fout.create_dataset(
                        '{}'.format(sentence_id),
                        concat_embedding_layers.shape,
                        dtype='float32',
                        data=concat_embedding_layers
                    )
                    sentence_id += 1 

def __call__(self, x, active_len, mask, **kwargs):
        """__call__(self, x, finetune=False)

        Invokes the forward propagation of BatchNormalization.

        In training mode, the BatchNormalization computes moving averages of
        mean and variance for evaluation during training, and normalizes the
        input using batch statistics.

        .. warning::

           ``test`` argument is not supported anymore since v2.
           Instead, use ``chainer.using_config('train', False)``.
           See :func:`chainer.using_config`.

        Args:
            x (Variable): Input variable.
            finetune (bool): If it is in the training mode and ``finetune`` is
                ``True``, BatchNormalization runs in fine-tuning mode; it
                accumulates the input array to compute population statistics
                for normalization, and normalizes the input using batch
                statistics.

        """
        argument.check_unexpected_kwargs(
            kwargs, test='test argument is not supported anymore. '
            'Use chainer.using_config')
        finetune, = argument.parse_kwargs(kwargs, ('finetune', False))

        if hasattr(self, 'gamma'):
            gamma = self.gamma
        else:
            with cuda.get_device_from_id(self._device_id):
                gamma = variable.Variable(self.xp.ones(
                    self.avg_mean.shape, dtype=x.dtype))

        if hasattr(self, 'beta'):
            beta = self.beta
        else:
            with cuda.get_device_from_id(self._device_id):
                beta = variable.Variable(self.xp.zeros(
                    self.avg_mean.shape, dtype=x.dtype))

        if configuration.config.train:
            if finetune:
                self.N += 1
                decay = 1. - 1. / self.N
            else:
                decay = self.decay

            ret = func_active_bn.batch_normalization(
                x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
                running_var=self.avg_var, decay=decay, active_len=active_len,
                mask=mask)
        else:
            # Use running average statistics or fine-tuned statistics.
            mean = variable.Variable(self.avg_mean)
            var = variable.Variable(self.avg_var)
            ret = func_active_bn.fixed_batch_normalization(
                x, gamma, beta, mean, var, self.eps)
        return ret 

def train():
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', '-g', type=int, default=-1)
    parser.add_argument('--model', '-m', type=str, default=None)
    parser.add_argument('--opt', type=str, default=None)
    parser.add_argument('--epoch', '-e', type=int, default=3)
    parser.add_argument('--lr', '-l', type=float, default=0.001)
    parser.add_argument('--inf', type=int, default=10)
    parser.add_argument('--outf', type=int, default=10)
    parser.add_argument('--batch', '-b', type=int, default=8)
    args = parser.parse_args()

    train = dataset.MovingMnistDataset(0, 7000, args.inf, args.outf)
    train_iter = iterators.SerialIterator(train, batch_size=args.batch, shuffle=True)
    test = dataset.MovingMnistDataset(7000, 10000, args.inf, args.outf)
    test_iter = iterators.SerialIterator(test, batch_size=args.batch, repeat=False, shuffle=False)

    model = network.MovingMnistNetwork(sz=[128,64,64], n=2)

    if args.model != None:
        print( "loading model from " + args.model )
        serializers.load_npz(args.model, model)

    if args.gpu >= 0:
        cuda.get_device_from_id(0).use()
        model.to_gpu()

    opt = optimizers.Adam(alpha=args.lr)
    opt.setup(model)

    if args.opt != None:
        print( "loading opt from " + args.opt )
        serializers.load_npz(args.opt, opt)

    updater = training.StandardUpdater(train_iter, opt, device=args.gpu)
    trainer = training.Trainer(updater, (args.epoch, 'epoch'), out='results')

    trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu))
    trainer.extend(extensions.LogReport(trigger=(10, 'iteration')))
    trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'validation/main/loss']))
    trainer.extend(extensions.ProgressBar(update_interval=1))

    trainer.run()

    modelname = "./results/model"
    print( "saving model to " + modelname )
    serializers.save_npz(modelname, model)

    optname = "./results/opt"
    print( "saving opt to " + optname )
    serializers.save_npz(optname, opt)