Python numpy.std() Examples

def get_graph_stats(graph_obj_handle, prop='degrees'):
    # if prop == 'degrees':
    num_cores = multiprocessing.cpu_count()
    inputs = [int(i*len(graph_obj_handle)/num_cores) for i in range(num_cores)] + [len(graph_obj_handle)]
    res = Parallel(n_jobs=num_cores)(delayed(get_values)(graph_obj_handle, inputs[i], inputs[i+1], prop) for i in range(num_cores))

    stat_dict = {}

    if 'degrees' in prop:
        stat_dict['degrees'] = list(set([d for core_res in res for file_res in core_res for d in file_res['degrees']]))
    if 'edge_labels' in prop:
        stat_dict['edge_labels'] = list(set([d for core_res in res for file_res in core_res for d in file_res['edge_labels']]))
    if 'target_mean' in prop or 'target_std' in prop:
        param = np.array([file_res['params'] for core_res in res for file_res in core_res])
    if 'target_mean' in prop:
        stat_dict['target_mean'] = np.mean(param, axis=0)
    if 'target_std' in prop:
        stat_dict['target_std'] = np.std(param, axis=0)

    return stat_dict 

def reverse_generator(generator, X_sample, y_sample, title):
    """Gradient descent to map images back to their latent vectors."""

    latent_vec = np.random.normal(size=(1, 100))

    # Function for figuring out how to bump the input.
    target = K.placeholder()
    loss = K.sum(K.square(generator.outputs[0] - target))
    grad = K.gradients(loss, generator.inputs[0])[0]
    update_fn = K.function(generator.inputs + [target], [grad])

    # Repeatedly apply the update rule.
    xs = []
    for i in range(60):
        print('%d: latent_vec mean=%f, std=%f'
              % (i, np.mean(latent_vec), np.std(latent_vec)))
        xs.append(generator.predict_on_batch([latent_vec, y_sample]))
        for _ in range(10):
            update_vec = update_fn([latent_vec, y_sample, X_sample])[0]
            latent_vec -= update_vec * update_rate

    # Plots the samples.
    xs = np.concatenate(xs, axis=0)
    plot_as_gif(xs, X_sample, title) 

def sigma_bin_walls(sigma, bins):
        import scipy, scipy.cluster, scipy.cluster.vq as vq
        std = np.std(sigma)
        if np.isclose(std, 0): return pimms.imm_array([0, np.max(sigma)])
        cl = sorted(std * vq.kmeans(sigma/std, bins)[0])
        cl = np.mean([cl[:-1],cl[1:]], axis=0)
        return pimms.imm_array(np.concatenate(([0], cl, [np.max(sigma)]))) 

def __init__(self, bam, keepReads=False):
        self.insertSizes = []
        self.readLengths = []
        self.orientations = []
        self._insertSizeKDE = None
        self.singleEnded = False

        self._insertSizeScores = {} # cache

        try:
            self.insertSizes, self.reads, self.orientations, self.readLengths = sampleInsertSizes(bam, keepReads=keepReads)
            if len(self.insertSizes) > 1:
                logging.info("  insert size mean: {:.2f} std: {:.2f}".format(numpy.mean(self.insertSizes), numpy.std(self.insertSizes)))
        except ValueError as e:
            print("*"*100, "here")
            print("ERROR:", e) 

def upper_bollinger_band(data, period, std_mult=2.0):
    """
    Upper Bollinger Band.

    Formula:
    u_bb = SMA(t) + STD(SMA(t-n:t)) * std_mult
    """
    check_for_period_error(data, period)

    period = int(period)
    simple_ma = sma(data, period)[period-1:]

    upper_bb = []
    for idx in range(len(data) - period + 1):
        std_dev = np.std(data[idx:idx + period])
        upper_bb.append(simple_ma[idx] + std_dev * std_mult)
    upper_bb = fill_for_noncomputable_vals(data, upper_bb)

    return np.array(upper_bb) 

def lower_bollinger_band(data, period, std=2.0):
    """
    Lower Bollinger Band.

    Formula:
    u_bb = SMA(t) - STD(SMA(t-n:t)) * std_mult
    """
    check_for_period_error(data, period)

    period = int(period)
    simple_ma = sma(data, period)[period-1:]

    lower_bb = []
    for idx in range(len(data) - period + 1):
        std_dev = np.std(data[idx:idx + period])
        lower_bb.append(simple_ma[idx] - std_dev * std)
    lower_bb = fill_for_noncomputable_vals(data, lower_bb)

    return np.array(lower_bb) 

def bandwidth(data, period, std=2.0):
    """
    Bandwidth.

    Formula:
    bw = u_bb - l_bb / m_bb
    """
    check_for_period_error(data, period)

    period = int(period)
    bandwidth = ((upper_bollinger_band(data, period, std) -
                 lower_bollinger_band(data, period, std)) /
                 middle_bollinger_band(data, period, std)
                 )

    return bandwidth 

def standard_deviation(data, period):
    """
    Standard Deviation.

    Formula:
    std = sqrt(avg(abs(x - avg(x))^2))
    """
    check_for_period_error(data, period)

    stds = list(map(
        lambda idx:
        np.std(data[idx+1-period:idx+1], ddof=1),
        range(period-1, len(data))
        ))

    stds = fill_for_noncomputable_vals(data, stds)
    return stds 

def log(self):
        end_idxs = np.nonzero(self._dones)[0] + 1

        returns = []

        start_idx = 0
        for end_idx in end_idxs:
            rewards = self._rewards[start_idx:end_idx]
            returns.append(np.sum(rewards))

            start_idx = end_idx

        logger.record_tabular('ReturnAvg', np.mean(returns))
        logger.record_tabular('ReturnStd', np.std(returns))
        logger.record_tabular('ReturnMin', np.min(returns))
        logger.record_tabular('ReturnMax', np.max(returns))

##################
### Tensorflow ###
################## 

def update_critic(self, ob_no, hidden, q_n):
        """
        given:
            self.num_value_iters
            self.l2_reg

        arguments:
            ob_no: (minibsize, history, meta_obs_dim)
            hidden: (minibsize, self.gru_size)
            q_n: (minibsize)

        requires:
            self.num_value_iters
        """
        target_n = (q_n - np.mean(q_n))/(np.std(q_n)+1e-8)
        for k in range(self.num_value_iters):
            critic_loss, _ = self.sess.run(
                [self.critic_loss, self.critic_update_op],
                feed_dict={self.sy_target_n: target_n, self.sy_ob_no: ob_no, self.sy_hidden: hidden})
        return critic_loss 

def get(self):
        tpr, fpr, accuracy, threshold = calculate_roc(
            self.thresholds, np.asarray(
                self.dists), np.asarray(
                self.issame), self.nfolds)

        val, val_std, far = calculate_val(
            self.thresholds, np.asarray(
                self.dists), np.asarray(
                self.issame), self.far_target, self.nfolds)

        acc, acc_std = np.mean(accuracy), np.std(accuracy)
        threshold = (
                1 - threshold) if self.dist_type == 'cosine' else threshold
        return tpr, fpr, acc, threshold, val, val_std, far, acc_std


# code below is modified from project <Facenet (David Sandberg)> and
# <Gluon-Face> 

def extractMeanDataStats(size = [200, 200, 100], 
						postfix = '_200x200x100orig', 
						main_folder_path = '../../Data/MS2017b/', 
						):
	scan_folders = glob.glob(main_folder_path + 'scans/*')
	img_path = 'pre/FLAIR' + postfix + '.nii.gz'
	segm_path = 'wmh' + postfix + '.nii.gz'
	
	shape_ = [len(scan_folders), size[0], size[1], size[2]]
	arr = np.zeros(shape_)

	for i, sf in enumerate(scan_folders):
		arr[i, :,:,:] =  numpyFromScan(os.path.join(sf,img_path)).squeeze()

	arr /= len(scan_folders)

	means = np.mean(arr)
	stds = np.std(arr, axis = 0)

	np.save(main_folder_path + 'extra_data/std' + postfix, stds)
	np.save(main_folder_path + 'extra_data/mean' + postfix, means) 

def test_not_none(self):
        mech = std
        self.assertIsNotNone(mech) 

def bias_variance_decomposition(self, graphs, targets,
                                    cv=5, n_bootstraps=10):
        """bias_variance_decomposition."""
        x = self.transform(graphs)
        score_list = []
        for i in range(n_bootstraps):
            scores = cross_val_score(
                self.model, x, targets, cv=cv)
            score_list.append(scores)
        score_list = np.array(score_list)
        mean_scores = np.mean(score_list, axis=1)
        std_scores = np.std(score_list, axis=1)
        return mean_scores, std_scores 

def applyInSubjectNormalize(img):
	m = np.mean(img[img != 0])
	s = np.std(img[img != 0])
	img = (img - m) / s
	return img 

def test_gaussian_noise():
    np.random.seed(2012)
    for shape in [10, 10], [100, 1000], [500, 50]:
        val = np.random.normal(0, 10)
        x = np.full(shape, val)
        noisy = gaussian_noise(x, np.ones(shape[1]))
        assert (np.mean(noisy) - np.mean(x)) ** 2 < 1.0
        assert (np.std(noisy) - 1.0) ** 2 < 0.1
        noisy = gaussian_noise(x, np.full(shape[1], 10.0))
        assert (np.std(noisy) - 10.0) ** 2 < 0.1 

def __init__(self, expert_path, train_fraction=0.7, traj_limitation=-1, randomize=True):
        traj_data = np.load(expert_path)
        if traj_limitation < 0:
            traj_limitation = len(traj_data['obs'])
        obs = traj_data['obs'][:traj_limitation]
        acs = traj_data['acs'][:traj_limitation]

        def flatten(x):
            # x.shape = (E,), or (E, L, D)
            _, size = x[0].shape
            episode_length = [len(i) for i in x]
            y = np.zeros((sum(episode_length), size))
            start_idx = 0
            for l, x_i in zip(episode_length, x):
                y[start_idx:(start_idx+l)] = x_i
                start_idx += l
                return y
        self.obs = np.array(flatten(obs))
        self.acs = np.array(flatten(acs))
        self.rets = traj_data['ep_rets'][:traj_limitation]
        self.avg_ret = sum(self.rets)/len(self.rets)
        self.std_ret = np.std(np.array(self.rets))
        if len(self.acs) > 2:
            self.acs = np.squeeze(self.acs)
        assert len(self.obs) == len(self.acs)
        self.num_traj = min(traj_limitation, len(traj_data['obs']))
        self.num_transition = len(self.obs)
        self.randomize = randomize
        self.dset = Dset(self.obs, self.acs, self.randomize)
        # for behavior cloning
        self.train_set = Dset(self.obs[:int(self.num_transition*train_fraction), :],
                              self.acs[:int(self.num_transition*train_fraction), :],
                              self.randomize)
        self.val_set = Dset(self.obs[int(self.num_transition*train_fraction):, :],
                            self.acs[int(self.num_transition*train_fraction):, :],
                            self.randomize)
        self.log_info() 

def compute_all_metrics_statistics(all_results):
  """Computes statistics of metrics across multiple decodings."""
  statistics = {}
  for key in all_results[0].keys():
    values = [result[key] for result in all_results]
    values = np.vstack(values)
    statistics[key + "_MEAN"] = np.mean(values, axis=0)
    statistics[key + "_STD"] = np.std(values, axis=0)
    statistics[key + "_MIN"] = np.min(values, axis=0)
    statistics[key + "_MAX"] = np.max(values, axis=0)
  return statistics 

def applyGlobalNormalize(img, postfix, main_folder_path = '../../Data/MS2017b/'):
	#subtract by dataset mean and divide by pixel standard deviation
	means = np.load(main_folder_path + 'extra_data/mean' + postfix + '.npy')
	stds = np.load(main_folder_path + 'extra_data/std' + postfix + '.npy')
	img = (img - means) / (stds + 0.000001)
	return img 

def get_inception_score(images, splits=10, get_split=False):
    assert (type(images) == list)
    assert (type(images[0]) == np.ndarray)
    assert (len(images[0].shape) == 3)
    assert (np.max(images[0]) > 10)
    assert (np.min(images[0]) >= 0.0)
    inps = []
    for img in images:
        img = img.astype(np.float32)
        inps.append(np.expand_dims(img, 0))
    bs = 100
    with tf.Session() as sess:
        preds = []
        n_batches = int(math.ceil(float(len(inps)) / float(bs)))
        for i in range(n_batches):
            sys.stdout.write(".")
            sys.stdout.flush()
            inp = inps[(i * bs):min((i + 1) * bs, len(inps))]
            inp = np.concatenate(inp, 0)
            pred = sess.run(softmax, {'ExpandDims:0': inp})
            preds.append(pred)
        preds = np.concatenate(preds, 0)
        scores = []
        objectness = []
        diversity = []
        for i in range(splits):
            part = preds[(i * preds.shape[0] // splits):((i + 1) * preds.shape[0] // splits), :]
            kl = part * (np.log(part) - np.log(np.expand_dims(np.mean(part, 0), 0)))
            o = -part * np.log(part)
            d = -part * np.log(np.expand_dims(np.mean(part, 0), 0))
            kl = np.mean(np.sum(kl, 1))
            objectness.append(np.exp(np.mean(np.sum(o, 1))))
            diversity.append(np.exp(np.mean(np.sum(d, 1))))
            scores.append(np.exp(kl))
        if get_split:
            return np.mean(scores), np.std(scores), np.mean(objectness), np.mean(diversity)
        return np.mean(scores), np.std(scores)


# This function is called automatically. 

def estimate_advantage(self, ob_no, next_ob_no, re_n, terminal_n):
        """
            Estimates the advantage function value for each timestep.

            let sum_of_path_lengths be the sum of the lengths of the paths sampled from 
                Agent.sample_trajectories

            arguments:
                ob_no: shape: (sum_of_path_lengths, ob_dim)
                next_ob_no: shape: (sum_of_path_lengths, ob_dim). The observation after taking one step forward
                re_n: length: sum_of_path_lengths. Each element in re_n is a scalar containing
                    the reward for each timestep
                terminal_n: length: sum_of_path_lengths. Each element in terminal_n is either 1 if the episode ended
                    at that timestep of 0 if the episode did not end

            returns:
                adv_n: shape: (sum_of_path_lengths). A single vector for the estimated 
                    advantages whose length is the sum of the lengths of the paths
        """
        next_values_n = self.sess.run(self.critic_prediction, feed_dict={self.sy_ob_no: next_ob_no})
        q_n = re_n + self.gamma * next_values_n * (1-terminal_n)
        curr_values_n = self.sess.run(self.critic_prediction, feed_dict={self.sy_ob_no: ob_no})
        adv_n = q_n - curr_values_n

        if self.normalize_advantages:
            adv_n = (adv_n - np.mean(adv_n)) / (np.std(adv_n) + 1e-8)
        return adv_n 

def sample_action(self, policy_parameters):
        """ Constructs a symbolic operation for stochastically sampling from the policy
            distribution

            arguments:
                policy_parameters
                    if discrete: logits of a categorical distribution over actions 
                        sy_logits_na: (batch_size, self.ac_dim)
                    if continuous: (mean, log_std) of a Gaussian distribution over actions
                        sy_mean: (batch_size, self.ac_dim)
                        sy_logstd: (self.ac_dim,)

            returns:
                sy_sampled_ac: 
                    if discrete: (batch_size)
                    if continuous: (batch_size, self.ac_dim)

            Hint: for the continuous case, use the reparameterization trick:
                 The output from a Gaussian distribution with mean 'mu' and std 'sigma' is
        
                      mu + sigma * z,         z ~ N(0, I)
        
                 This reduces the problem to just sampling z. (Hint: use tf.random_normal!)
        """
        if self.discrete:
            sy_logits_na = policy_parameters
            sy_sampled_ac = tf.squeeze(tf.multinomial(sy_logits_na, num_samples=1), axis=1)
        else:
            sy_mean, sy_logstd = policy_parameters
            sy_sampled_ac = sy_mean + tf.exp(sy_logstd) * tf.random_normal(tf.shape(sy_mean), 0, 1)
        return sy_sampled_ac 

def test_no_params(self):
        a = np.array([1, 2, 3])
        with self.assertWarns(PrivacyLeakWarning):
            res = std(a)
        self.assertIsNotNone(res) 

def build_policy(x, h, output_size, scope, n_layers, size, gru_size, recurrent=True, activation=tf.tanh, output_activation=None):
    """
    build recurrent policy

    arguments:
        x: placeholder variable for the input, which has dimension (batch_size, history, input_size)
        h: placeholder variable for the hidden state, which has dimension (batch_size, gru_size)
        output_size: size of the output layer, same as action dimension
        scope: variable scope of the network
        n_layers: number of hidden layers (not counting recurrent units)
        size: dimension of the hidden layer in the encoder
        gru_size: dimension of the recurrent hidden state if there is one
        recurrent: if the network should be recurrent or feedforward
        activation: activation of the hidden layers
        output_activation: activation of the ouput layers

    returns:
        output placeholder of the network (the result of a forward pass)

    n.b. we predict both the mean and std of the gaussian policy, and we don't want the std to start off too large
    initialize the last layer of the policy with a guassian init of mean 0 and std 0.01
    """
    with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
        if recurrent:
            x, h = build_rnn(x, h, gru_size, scope, n_layers, size, activation=activation, output_activation=activation)
        else:
            x = tf.reshape(x, (-1, x.get_shape()[1]*x.get_shape()[2]))
            x = build_mlp(x, gru_size, scope, n_layers + 1, size, activation=activation, output_activation=activation)
        x = tf.layers.dense(x, output_size, activation=output_activation, kernel_initializer=tf.initializers.truncated_normal(mean=0.0, stddev=0.01), bias_initializer=tf.zeros_initializer(), name='decoder')
    return x, h 

def test_no_epsilon(self):
        a = np.array([1, 2, 3])
        self.assertIsNotNone(std(a, bounds=(0, 1))) 

def estimate_return(self, ob_no, re_n):
        """
            Estimates the returns over a set of trajectories.

            let sum_of_path_lengths be the sum of the lengths of the paths sampled from 
                Agent.sample_trajectories
            let num_paths be the number of paths sampled from Agent.sample_trajectories

            arguments:
                ob_no: shape: (sum_of_path_lengths, ob_dim)
                re_n: length: num_paths. Each element in re_n is a numpy array 
                    containing the rewards for the particular path

            returns:
                q_n: shape: (sum_of_path_lengths). A single vector for the estimated q values 
                    whose length is the sum of the lengths of the paths
                adv_n: shape: (sum_of_path_lengths). A single vector for the estimated 
                    advantages whose length is the sum of the lengths of the paths
        """
        q_n = self.sum_of_rewards(re_n)
        adv_n = self.compute_advantage(ob_no, q_n)
        #====================================================================================#
        #                           ----------PROBLEM 3----------
        # Advantage Normalization
        #====================================================================================#
        if self.normalize_advantages:
            # On the next line, implement a trick which is known empirically to reduce variance
            # in policy gradient methods: normalize adv_n to have mean zero and std=1.
            # YOUR_CODE_HERE
            adv_n = normalize(adv_n)
        return q_n, adv_n 

def compute_advantage(self, ob_no, q_n):
        """
            Computes advantages by (possibly) subtracting a baseline from the estimated Q values

            let sum_of_path_lengths be the sum of the lengths of the paths sampled from 
                Agent.sample_trajectories
            let num_paths be the number of paths sampled from Agent.sample_trajectories

            arguments:
                ob_no: shape: (sum_of_path_lengths, ob_dim)
                q_n: shape: (sum_of_path_lengths). A single vector for the estimated q values 
                    whose length is the sum of the lengths of the paths

            returns:
                adv_n: shape: (sum_of_path_lengths). A single vector for the estimated 
                    advantages whose length is the sum of the lengths of the paths
        """
        #====================================================================================#
        #                           ----------PROBLEM 6----------
        # Computing Baselines
        #====================================================================================#
        if self.nn_baseline:
            # If nn_baseline is True, use your neural network to predict reward-to-go
            # at each timestep for each trajectory, and save the result in a variable 'b_n'
            # like 'ob_no', 'ac_na', and 'q_n'.
            #
            # Hint #bl1: rescale the output from the nn_baseline to match the statistics
            # (mean and std) of the current batch of Q-values. (Goes with Hint
            # #bl2 in Agent.update_parameters.
            # YOUR CODE HERE
            b_n = self.sess.run(self.baseline_prediction, feed_dict={self.sy_ob_no:ob_no})
            b_n = normalize(b_n, q_n.mean(), q_n.std())
            adv_n = q_n - b_n
        else:
            adv_n = q_n.copy()
        return adv_n 

def sample_action(self, policy_parameters):
        """ Constructs a symbolic operation for stochastically sampling from the policy
            distribution

            arguments:
                policy_parameters
                    if discrete: logits of a categorical distribution over actions 
                        sy_logits_na: (batch_size, self.ac_dim)
                    if continuous: (mean, log_std) of a Gaussian distribution over actions
                        sy_mean: (batch_size, self.ac_dim)
                        sy_logstd: (self.ac_dim,)

            returns:
                sy_sampled_ac: 
                    if discrete: (batch_size,)
                    if continuous: (batch_size, self.ac_dim)

            Hint: for the continuous case, use the reparameterization trick:
                 The output from a Gaussian distribution with mean 'mu' and std 'sigma' is
        
                      mu + sigma * z,         z ~ N(0, I)
        
                 This reduces the problem to just sampling z. (Hint: use tf.random_normal!)
        """
        if self.discrete:
            sy_logits_na = policy_parameters
            # YOUR_CODE_HERE
            sy_sampled_ac = tf.squeeze(tf.multinomial(sy_logits_na, 1), axis=1)
        else:
            sy_mean, sy_logstd = policy_parameters
            # YOUR_CODE_HERE
            sy_sampled_ac = sy_mean + tf.exp(sy_logstd) * tf.random_normal(tf.shape(sy_mean))
        return sy_sampled_ac

    #========================================================================================#
    #                           ----------PROBLEM 2----------
    #========================================================================================# 

def sample_action(self, policy_parameters):
        """ Constructs a symbolic operation for stochastically sampling from the policy
            distribution

            arguments:
                policy_parameters
                    if discrete: logits of a categorical distribution over actions 
                        sy_logits_na: (batch_size, self.ac_dim)
                    if continuous: (mean, log_std) of a Gaussian distribution over actions
                        sy_mean: (batch_size, self.ac_dim)
                        sy_logstd: (self.ac_dim,)

            returns:
                sy_sampled_ac: 
                    if discrete: (batch_size)
                    if continuous: (batch_size, self.ac_dim)

            Hint: for the continuous case, use the reparameterization trick:
                 The output from a Gaussian distribution with mean 'mu' and std 'sigma' is
        
                      mu + sigma * z,         z ~ N(0, I)
        
                 This reduces the problem to just sampling z. (Hint: use tf.random_normal!)
        """
        if self.discrete:
            sy_logits_na = policy_parameters
            # YOUR_HW2 CODE_HERE
            sy_sampled_ac = tf.squeeze(tf.multinomial(sy_logits_na, 1), axis=1)
        else:
            sy_mean, sy_logstd = policy_parameters
            # YOUR_HW2 CODE_HERE
            sy_sampled_ac = sy_mean + tf.exp(sy_logstd) * tf.random_normal(tf.shape(sy_mean))
        return sy_sampled_ac 

def train_pca(self, data, mu_data, std_data, subspace_dim):
        '''
        Modified PCA function, different from the one in sklearn
        :param data: data matrix
        :param mu_data: mu
        :param std_data: std
        :param subspace_dim: dim
        :return: a wrapped machine object
        '''
        t = bob.learn.linear.PCATrainer()
        machine, variances = t.train(data)

        # For re-shaping, we need to copy...
        variances = variances.copy()

        # compute variance percentage, if desired
        if isinstance(subspace_dim, float):
            cummulated = np.cumsum(variances) / np.sum(variances)
            for index in range(len(cummulated)):
                if cummulated[index] > subspace_dim:
                    subspace_dim = index
                    break
            subspace_dim = index
        machine.resize(machine.shape[0], subspace_dim)
        machine.input_subtract = mu_data
        machine.input_divide = std_data

        return machine