Java Code Examples for org.nd4j.linalg.ops.transforms.Transforms#pow()

private INDArray activateHelperFullArray(INDArray inputArray, int[] poolDim) {
    switch (poolingType) {
        case MAX:
            return inputArray.max(poolDim);
        case AVG:
            return inputArray.mean(poolDim);
        case SUM:
            return inputArray.sum(poolDim);
        case PNORM:
            //P norm: https://arxiv.org/pdf/1311.1780.pdf
            //out = (1/N * sum( |in| ^ p) ) ^ (1/p)
            int pnorm = layerConf().getPnorm();

            INDArray abs = Transforms.abs(inputArray, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(poolDim);

            return Transforms.pow(pNorm, 1.0 / pnorm, false);
        default:
            throw new RuntimeException("Unknown or not supported pooling type: " + poolingType + " " + layerId());
    }
}
 

/**
 * Return a reduced basis set that covers a certain fraction of the variance of the data
 * @param variance The desired fractional variance (0 to 1), it will always be greater than the value.
 * @return The basis vectors as columns, size <i>N</i> rows by <i>ndims</i> columns, where <i>ndims</i> is less than or equal to <i>N</i>
 */
public INDArray reducedBasis(double variance) {
    INDArray vars = Transforms.pow(eigenvalues, -0.5, true);
    double res = vars.sumNumber().doubleValue();
    double total = 0.0;
    int ndims = 0;
    for (int i = 0; i < vars.columns(); i++) {
        ndims++;
        total += vars.getDouble(i);
        if (total / res > variance)
            break;
    }
    INDArray result = Nd4j.create(eigenvectors.rows(), ndims);
    for (int i = 0; i < ndims; i++)
        result.putColumn(i, eigenvectors.getColumn(i));
    return result;
}
 

/**
 * This method performs a dimensionality reduction, including principal components
 * that cover a fraction of the total variance of the system.  It does all calculations
 * about the mean.
 * @param in A matrix of datapoints as rows, where column are features with fixed number N
 * @param variance The desired fraction of the total variance required
 * @return The reduced basis set
 */
public static INDArray pca2(INDArray in, double variance) {
    // let's calculate the covariance and the mean
    INDArray[] covmean = covarianceMatrix(in);
    // use the covariance matrix (inverse) to find "force constants" and then break into orthonormal
    // unit vector components
    INDArray[] pce = principalComponents(covmean[0]);
    // calculate the variance of each component
    INDArray vars = Transforms.pow(pce[1], -0.5, true);
    double res = vars.sumNumber().doubleValue();
    double total = 0.0;
    int ndims = 0;
    for (int i = 0; i < vars.columns(); i++) {
        ndims++;
        total += vars.getDouble(i);
        if (total / res > variance)
            break;
    }
    INDArray result = Nd4j.create(in.columns(), ndims);
    for (int i = 0; i < ndims; i++)
        result.putColumn(i, pce[0].getColumn(i));
    return result;
}
 

/**
 * Return a reduced basis set that covers a certain fraction of the variance of the data
 * @param variance The desired fractional variance (0 to 1), it will always be greater than the value.
 * @return The basis vectors as columns, size <i>N</i> rows by <i>ndims</i> columns, where <i>ndims</i> is less than or equal to <i>N</i>
 */
public INDArray reducedBasis(double variance) {
    INDArray vars = Transforms.pow(eigenvalues, -0.5, true);
    double res = vars.sumNumber().doubleValue();
    double total = 0.0;
    int ndims = 0;
    for (int i = 0; i < vars.columns(); i++) {
        ndims++;
        total += vars.getDouble(i);
        if (total / res > variance)
            break;
    }
    INDArray result = Nd4j.create(eigenvectors.rows(), ndims);
    for (int i = 0; i < ndims; i++)
        result.putColumn(i, eigenvectors.getColumn(i));
    return result;
}
 

@Override
public INDArray gradient(INDArray x, INDArray preOutDistributionParams) {
    INDArray output = preOutDistributionParams.dup();
    activationFn.getActivation(output, true);

    val size = output.size(1) / 2;
    INDArray mean = output.get(NDArrayIndex.all(), NDArrayIndex.interval(0, size));
    INDArray logStdevSquared = output.get(NDArrayIndex.all(), NDArrayIndex.interval(size, 2 * size));

    INDArray sigmaSquared = Transforms.exp(logStdevSquared, true).castTo(x.dataType());

    INDArray xSubMean = x.sub(mean.castTo(x.dataType()));
    INDArray xSubMeanSq = xSubMean.mul(xSubMean);

    INDArray dLdmu = xSubMean.divi(sigmaSquared);

    INDArray sigma = Transforms.sqrt(sigmaSquared, true);
    INDArray sigma3 = Transforms.pow(sigmaSquared, 3.0 / 2);

    INDArray dLdsigma = sigma.rdiv(-1).addi(xSubMeanSq.divi(sigma3));
    INDArray dLdlogSigma2 = sigma.divi(2).muli(dLdsigma);

    INDArray dLdx = Nd4j.createUninitialized(preOutDistributionParams.dataType(), output.shape());
    dLdx.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, size)}, dLdmu);
    dLdx.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(size, 2 * size)}, dLdlogSigma2);
    dLdx.negi();

    //dL/dz
    return activationFn.backprop(preOutDistributionParams.dup(), dLdx).getFirst();
}
 

/**
 * Generates a set of <i>count</i> random samples with the same variance and mean and eigenvector/values
 * as the data set used to initialize the PCA object, with same number of features <i>N</i>.
 * @param count The number of samples to generate
 * @return A matrix of size <i>count</i> rows by <i>N</i> columns
 */
public INDArray generateGaussianSamples(long count) {
    INDArray samples = Nd4j.randn(new long[] {count, eigenvalues.columns()});
    INDArray factors = Transforms.pow(eigenvalues, -0.5, true);
    samples.muliRowVector(factors);
    return Nd4j.tensorMmul(eigenvectors, samples, new int[][] {{1}, {1}}).transposei().addiRowVector(mean);
}
 

/**
 * Estimate the variance of a single record with reduced # of dimensions.
 * @param data A single record with the same <i>N</i> features as the constructing data set
 * @param ndims The number of dimensions to include in calculation
 * @return The fraction (0 to 1) of the total variance covered by the <i>ndims</i> basis set.
 */
public double estimateVariance(INDArray data, int ndims) {
    INDArray dx = data.sub(mean);
    INDArray v = eigenvectors.transpose().mmul(dx.reshape(dx.columns(), 1));
    INDArray t2 = Transforms.pow(v, 2);
    double fraction = t2.get(NDArrayIndex.interval(0, ndims)).sumNumber().doubleValue();
    double total = t2.sumNumber().doubleValue();
    return fraction / total;
}
 

/**
 * Generates a set of <i>count</i> random samples with the same variance and mean and eigenvector/values
 * as the data set used to initialize the PCA object, with same number of features <i>N</i>.
 * @param count The number of samples to generate
 * @return A matrix of size <i>count</i> rows by <i>N</i> columns
 */
public INDArray generateGaussianSamples(long count) {
    INDArray samples = Nd4j.randn(new long[] {count, eigenvalues.columns()});
    INDArray factors = Transforms.pow(eigenvalues, -0.5, true);
    samples.muliRowVector(factors);
    return Nd4j.tensorMmul(eigenvectors, samples, new int[][] {{1}, {1}}).transposei().addiRowVector(mean);
}
 

@Test
    public void testActivationBackprop() {

        Activation[] afns = new Activation[]{
                Activation.TANH,
                Activation.SIGMOID,
                Activation.ELU,
                Activation.SOFTPLUS,
                Activation.SOFTSIGN,
                Activation.HARDTANH,
                Activation.CUBE,
                //WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
                Activation.RELU,            //JVM crash
                Activation.LEAKYRELU        //JVM crash
        };

        for (Activation a : afns) {

            SameDiff sd = SameDiff.create();
            INDArray inArr = Nd4j.linspace(-3, 3, 7);
            INDArray labelArr = Nd4j.linspace(-3, 3, 7).muli(0.5);
            SDVariable in = sd.var("in", inArr.dup());

//            System.out.println("inArr: " + inArr);

            INDArray outExp;
            SDVariable out;
            switch (a) {
                case ELU:
                    out = sd.nn().elu("out", in);
                    outExp = Transforms.elu(inArr, true);
                    break;
                case HARDTANH:
                    out = sd.nn().hardTanh("out", in);
                    outExp = Transforms.hardTanh(inArr, true);
                    break;
                case LEAKYRELU:
                    out = sd.nn().leakyRelu("out", in, 0.01);
                    outExp = Transforms.leakyRelu(inArr, true);
                    break;
                case RELU:
                    out = sd.nn().relu("out", in, 0.0);
                    outExp = Transforms.relu(inArr, true);
                    break;
                case SIGMOID:
                    out = sd.nn().sigmoid("out", in);
                    outExp = Transforms.sigmoid(inArr, true);
                    break;
                case SOFTPLUS:
                    out = sd.nn().softplus("out", in);
                    outExp = Transforms.softPlus(inArr, true);
                    break;
                case SOFTSIGN:
                    out = sd.nn().softsign("out", in);
                    outExp = Transforms.softsign(inArr, true);
                    break;
                case TANH:
                    out = sd.math().tanh("out", in);
                    outExp = Transforms.tanh(inArr, true);
                    break;
                case CUBE:
                    out = sd.math().cube("out", in);
                    outExp = Transforms.pow(inArr, 3, true);
                    break;
                default:
                    throw new RuntimeException(a.toString());
            }

            //Sum squared error loss:
            SDVariable label = sd.var("label", labelArr.dup());
            SDVariable diff = label.sub("diff", out);
            SDVariable sqDiff = diff.mul("sqDiff", diff);
            SDVariable totSum = sd.sum("totSum", sqDiff, Integer.MAX_VALUE);    //Loss function...

            Map<String,INDArray> m = sd.output(Collections.emptyMap(), "out");
            INDArray outAct = m.get("out");
            assertEquals(a.toString(), outExp, outAct);

            // L = sum_i (label - out)^2
            //dL/dOut = 2(out - label)
            INDArray dLdOutExp = outExp.sub(labelArr).mul(2);
            INDArray dLdInExp = a.getActivationFunction().backprop(inArr.dup(), dLdOutExp.dup()).getFirst();

            Map<String,INDArray> grads = sd.calculateGradients(null, "out", "in");
//            sd.execBackwards(Collections.emptyMap());
//            SameDiff gradFn = sd.getFunction("grad");
            INDArray dLdOutAct = grads.get("out");
            INDArray dLdInAct = grads.get("in");

            assertEquals(a.toString(), dLdOutExp, dLdOutAct);
            assertEquals(a.toString(), dLdInExp, dLdInAct);
        }
    }
 

private INDArray epsilonHelperFullArray(INDArray inputArray, INDArray epsilon, int[] poolDim) {

        //Broadcast: occurs on the remaining dimensions, after the pool dimensions have been removed.
        //TODO find a more efficient way to do this
        int[] broadcastDims = new int[inputArray.rank() - poolDim.length];
        int count = 0;
        for (int i = 0; i < inputArray.rank(); i++) {
            if (ArrayUtils.contains(poolDim, i))
                continue;
            broadcastDims[count++] = i;
        }

        switch (poolingType) {
            case MAX:
                INDArray isMax = Nd4j.exec(new IsMax(inputArray, inputArray.ulike(), poolDim))[0];
                return Nd4j.getExecutioner().exec(new BroadcastMulOp(isMax, epsilon, isMax, broadcastDims));
            case AVG:
                //if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
                int n = 1;
                for (int d : poolDim) {
                    n *= inputArray.size(d);
                }
                INDArray ret = inputArray.ulike();
                Nd4j.getExecutioner().exec(new BroadcastCopyOp(ret, epsilon, ret, broadcastDims));
                ret.divi(n);

                return ret;
            case SUM:
                INDArray retSum = inputArray.ulike();
                Nd4j.getExecutioner().exec(new BroadcastCopyOp(retSum, epsilon, retSum, broadcastDims));
                return retSum;
            case PNORM:
                int pnorm = layerConf().getPnorm();

                //First: do forward pass to get pNorm array
                INDArray abs = Transforms.abs(inputArray, true);
                Transforms.pow(abs, pnorm, false);

                INDArray pNorm = Transforms.pow(abs.sum(poolDim), 1.0 / pnorm);

                //dL/dIn = dL/dOut * dOut/dIn
                //dOut/dIn = in .* |in|^(p-2) /  ||in||_p^(p-1), where ||in||_p is the output p-norm

                INDArray numerator;
                if (pnorm == 2) {
                    numerator = inputArray.dup();
                } else {
                    INDArray absp2 = Transforms.pow(Transforms.abs(inputArray, true), pnorm - 2, false);
                    numerator = inputArray.mul(absp2);
                }

                INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
                denom.rdivi(epsilon);
                Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, broadcastDims));

                return numerator;
            default:
                throw new RuntimeException("Unknown or not supported pooling type: " + poolingType + " " + layerId());
        }
    }
 

public static INDArray maskedPoolingTimeSeries(PoolingType poolingType, INDArray toReduce, INDArray mask,
                int pnorm, DataType dataType) {
    if (toReduce.rank() != 3) {
        throw new IllegalArgumentException("Expect rank 3 array: got " + toReduce.rank());
    }
    if (mask.rank() != 2) {
        throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
    }

    toReduce = toReduce.castTo(dataType);
    mask = mask.castTo(dataType);

    //Sum pooling: easy. Multiply by mask, then sum as normal
    //Average pooling: as above, but do a broadcast element-wise divi by mask.sum(1)
    //Max pooling: set to -inf if mask is 0, then do max as normal

    switch (poolingType) {
        case MAX:
            INDArray negInfMask = mask.castTo(dataType).rsub(1.0);
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, 0, 2));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            return withInf.max(2);
        case AVG:
        case SUM:
            INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, 0, 2));
            INDArray summed = masked.sum(2);
            if (poolingType == PoolingType.SUM) {
                return summed;
            }

            INDArray maskCounts = mask.sum(1);
            summed.diviColumnVector(maskCounts);
            return summed;
        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, 0, 2));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(2);

            return Transforms.pow(pNorm, 1.0 / pnorm);
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
 

/**
 * Fit the given model
 * @param iterator the data to iterate oer
 */
public void fit(DataSetIterator iterator) {
    while (iterator.hasNext()) {
        DataSet next = iterator.next();
        runningTotal += next.numExamples();
        batchCount = next.getFeatures().size(0);
        if (mean == null) {
            //start with the mean and std of zero
            //column wise
            mean = next.getFeatures().mean(0);
            std = (batchCount == 1) ? Nd4j.zeros(mean.shape()) : Transforms.pow(next.getFeatures().std(0), 2);
            std.muli(batchCount);
        } else {
            // m_newM = m_oldM + (x - m_oldM)/m_n;
            // This only works if batch size is 1, m_newS = m_oldS + (x - m_oldM)*(x - m_newM);
            INDArray xMinusMean = next.getFeatures().subRowVector(mean);
            INDArray newMean = mean.add(xMinusMean.sum(0).divi(runningTotal));
            // Using http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf
            // for a version of calc variance when dataset is partitioned into two sample sets
            // Also described in https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm
            // delta = mean_B - mean_A; A is data seen so far, B is the current batch
            // M2 is the var*n
            // M2 = M2_A + M2_B + delta^2 * nA * nB/(nA+nB)
            INDArray meanB = next.getFeatures().mean(0);
            INDArray deltaSq = Transforms.pow(meanB.subRowVector(mean), 2);
            INDArray deltaSqScaled =
                            deltaSq.mul(((float) runningTotal - batchCount) * batchCount / (float) runningTotal);
            INDArray mtwoB = Transforms.pow(next.getFeatures().std(0), 2);
            mtwoB.muli(batchCount);
            std = std.add(mtwoB);
            std = std.add(deltaSqScaled);
            mean = newMean;
        }

    }
    std.divi(runningTotal);
    std = Transforms.sqrt(std);
    std.addi(Nd4j.scalar(Nd4j.EPS_THRESHOLD));
    if (std.min(1) == Nd4j.scalar(Nd4j.EPS_THRESHOLD))
        logger.info("API_INFO: Std deviation found to be zero. Transform will round upto epsilon to avoid nans.");
    iterator.reset();
}
 

/**
 * Add rows of data to the statistics
 *
 * @param data the matrix containing multiple rows of data to include
 * @param mask (optionally) the mask of the data, useful for e.g. time series
 */
public Builder add(@NonNull INDArray data, INDArray mask) {
    data = DataSetUtil.tailor2d(data, mask);

    // Using https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm
    if (data == null) {
        // Nothing to add. Either data is empty or completely masked. Just skip it, otherwise we will get
        // null pointer exceptions.
        return this;
    }
    INDArray mean = data.mean(0).reshape(1,data.size(1));
    INDArray variance = data.var(false, 0).reshape(1,data.size(1));
    long count = data.size(0);

    if (runningMean == null) {
        // First batch
        runningMean = mean;
        runningVariance = variance;
        runningCount = count;

        if (data.size(0) == 1) {
            //Handle edge case: currently, reduction ops may return the same array
            //But we don't want to modify this array in-place later
            runningMean = runningMean.dup();
            runningVariance = runningVariance.dup();
        }
    } else {
        // Update running variance
        INDArray deltaSquared = Transforms.pow(mean.subRowVector(runningMean), 2);
        INDArray mB = variance.muli(count);
        runningVariance.muli(runningCount).addiRowVector(mB)
                        .addiRowVector(deltaSquared
                                        .muli((float) (runningCount * count) / (runningCount + count)))
                        .divi(runningCount + count);

        // Update running count
        runningCount += count;

        // Update running mean
        INDArray xMinusMean = data.subRowVector(runningMean);
        runningMean.addi(xMinusMean.sum(0).divi(runningCount));
    }

    return this;
}
 

@Test
public void testPooling2D_Same() {
    int[] miniBatches = {1, 3, 5};
    int[] depths = {1, 3, 5};
    int[] inHeights = {5, 21};
    int[] inWidths = {5, 21};
    int[] strideH = {1, 2};
    int[] strideW = {1, 2};
    int[] sizeW = {1, 2, 3};
    int[] sizeH = {1, 2, 3};
    int[] padH = {0};
    int[] padW = {0};
    Pooling2D.Pooling2DType[] types = new Pooling2D.Pooling2DType[]{Pooling2D.Pooling2DType.PNORM, Pooling2D.Pooling2DType.AVG, Pooling2D.Pooling2DType.MAX};

    int cnt = 0;

    for (Pooling2D.Pooling2DType type : types) {
        log.info("Trying pooling type: [{}]", type);
        for (int m : miniBatches) {
            for (int d : depths) {
                for (int h : inHeights) {
                    for (int w : inWidths) {
                        for (int sh : strideH) {
                            for (int sw : strideW) {
                                for (int kh : sizeH) {
                                    for (int kw : sizeW) {

                                        INDArray in = Nd4j.linspace(1, (m * d * h * w), (m * d * h * w), Nd4j.defaultFloatingPointType()).reshape(new int[]{m, d, h, w});

                                        int[] outSize = getOutputSize(in, new int[]{kh, kw}, new int[]{sh, sw}, null, true);

                                        //Calculate padding for same mode:
                                        int pHTotal = (outSize[0]-1)*sh + kh - h;
                                        int pWTotal = (outSize[1]-1)*sw + kw - w;
                                        int padTop = pHTotal / 2;
                                        int padLeft = pWTotal / 2;

                                        INDArray col = Nd4j.create(new int[]{m, d, outSize[0], outSize[1], kh, kw}, 'c');
                                        INDArray col2 = col.permute(0, 1, 4, 5, 2, 3);
                                        //INDArray col = Nd4j.createUninitialized(new int[]{m, d, kH, kW, outSize[0], outSize[1]}, 'c');
                                        //INDArray col2 = col;

                                        Convolution.im2col(in, kh, kw, sh, sw, padTop, padLeft, true, col2);

                                        INDArray col2d = col.reshape('c', m * d * outSize[0] * outSize[1], kh * kw);

                                        INDArray output = Nd4j.create(m, d, outSize[0], outSize[1]);



                                        INDArray reduced = null;
                                        switch (type) {
                                            case PNORM:
                                                int pnorm = 3;

                                                Transforms.abs(col2d, false);
                                                Transforms.pow(col2d, pnorm, false);
                                                reduced = col2d.sum(1);
                                                Transforms.pow(reduced, (1.0 / pnorm), false);

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.PNORM, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        (double) pnorm, outSize[0], outSize[1], output);

                                                break;
                                            case MAX:
                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.MAX, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.max(1);
                                                break;
                                            case AVG:

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.AVG, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.mean(1);
                                                break;
                                        }

                                        reduced = reduced.reshape('c',m,d, outSize[0], outSize[1]).dup('c');

                                        assertEquals("Failed opType: " + type, reduced, output);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
 

@Test
public void checkMeanVarianceEstimateCNNCompareModes() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    //Check that the internal global mean/variance estimate is approximately correct

    //First, Mnist data as 2d input (NOT taking into account convolution property)
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(Updater.RMSPROP).seed(12345).list()
            .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(false).build())
            .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER)
                    .activation(Activation.IDENTITY).nOut(10).build())
            .setInputType(InputType.convolutional(5, 5, 3)).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(Updater.RMSPROP).seed(12345).list()
            .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(true).build())
            .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER)
                    .activation(Activation.IDENTITY).nOut(10).build())
            .setInputType(InputType.convolutional(5, 5, 3)).build();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    int minibatch = 32;
    for (int i = 0; i < 10; i++) {
        DataSet ds = new DataSet(Nd4j.rand(new int[]{minibatch, 3, 5, 5}), Nd4j.rand(minibatch, 10));
        net.fit(ds);
        net2.fit(ds);

        INDArray globalVar = net.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_VAR);

        INDArray log10std = net2.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_LOG_STD);
        INDArray globalVar2 = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType());
        Transforms.pow(globalVar2, log10std, false);    // stdev = 10^(log10(stdev))
        globalVar2.muli(globalVar2);

        assertEquals(globalVar, globalVar2);
    }
}
 

/**
 * Fit the given model
 * @param iterator the data to iterate oer
 */
public void fit(DataSetIterator iterator) {
    while (iterator.hasNext()) {
        DataSet next = iterator.next();
        runningTotal += next.numExamples();
        batchCount = next.getFeatures().size(0);
        if (mean == null) {
            //start with the mean and std of zero
            //column wise
            mean = next.getFeatureMatrix().mean(0);
            std = (batchCount == 1) ? Nd4j.zeros(mean.shape()) : Transforms.pow(next.getFeatureMatrix().std(0), 2);
            std.muli(batchCount);
        } else {
            // m_newM = m_oldM + (x - m_oldM)/m_n;
            // This only works if batch size is 1, m_newS = m_oldS + (x - m_oldM)*(x - m_newM);
            INDArray xMinusMean = next.getFeatureMatrix().subRowVector(mean);
            INDArray newMean = mean.add(xMinusMean.sum(0).divi(runningTotal));
            // Using http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf
            // for a version of calc variance when dataset is partitioned into two sample sets
            // Also described in https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm
            // delta = mean_B - mean_A; A is data seen so far, B is the current batch
            // M2 is the var*n
            // M2 = M2_A + M2_B + delta^2 * nA * nB/(nA+nB)
            INDArray meanB = next.getFeatureMatrix().mean(0);
            INDArray deltaSq = Transforms.pow(meanB.subRowVector(mean), 2);
            INDArray deltaSqScaled =
                            deltaSq.mul(((float) runningTotal - batchCount) * batchCount / (float) runningTotal);
            INDArray mtwoB = Transforms.pow(next.getFeatureMatrix().std(0), 2);
            mtwoB.muli(batchCount);
            std = std.add(mtwoB);
            std = std.add(deltaSqScaled);
            mean = newMean;
        }

    }
    std.divi(runningTotal);
    std = Transforms.sqrt(std);
    std.addi(Nd4j.scalar(Nd4j.EPS_THRESHOLD));
    if (std.min(1) == Nd4j.scalar(Nd4j.EPS_THRESHOLD))
        logger.info("API_INFO: Std deviation found to be zero. Transform will round upto epsilon to avoid nans.");
    iterator.reset();
}
 

/**
 * Add rows of data to the statistics
 *
 * @param data the matrix containing multiple rows of data to include
 * @param mask (optionally) the mask of the data, useful for e.g. time series
 */
public Builder add(@NonNull INDArray data, INDArray mask) {
    data = DataSetUtil.tailor2d(data, mask);

    // Using https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm
    if (data == null) {
        // Nothing to add. Either data is empty or completely masked. Just skip it, otherwise we will get
        // null pointer exceptions.
        return this;
    }
    INDArray mean = data.mean(0);
    INDArray variance = data.var(false, 0);
    long count = data.size(0);

    if (runningMean == null) {
        // First batch
        runningMean = mean;
        runningVariance = variance;
        runningCount = count;

        if (data.size(0) == 1) {
            //Handle edge case: currently, reduction ops may return the same array
            //But we don't want to modify this array in-place later
            runningMean = runningMean.dup();
            runningVariance = runningVariance.dup();
        }
    } else {
        // Update running variance
        INDArray deltaSquared = Transforms.pow(mean.subRowVector(runningMean), 2);
        INDArray mB = variance.muli(count);
        runningVariance.muli(runningCount).addiRowVector(mB)
                        .addiRowVector(deltaSquared
                                        .muli((float) (runningCount * count) / (runningCount + count)))
                        .divi(runningCount + count);

        // Update running count
        runningCount += count;

        // Update running mean
        INDArray xMinusMean = data.subRowVector(runningMean);
        runningMean.addi(xMinusMean.sum(0).divi(runningCount));
    }

    return this;
}
 

@Test
public void testPooling2D_Same() {
    int[] miniBatches = {1, 3, 5};
    int[] depths = {1, 3, 5};
    int[] inHeights = {5, 21};
    int[] inWidths = {5, 21};
    int[] strideH = {1, 2};
    int[] strideW = {1, 2};
    int[] sizeW = {1, 2, 3};
    int[] sizeH = {1, 2, 3};
    int[] padH = {0};
    int[] padW = {0};
    Pooling2D.Pooling2DType[] types = new Pooling2D.Pooling2DType[]{Pooling2D.Pooling2DType.AVG, Pooling2D.Pooling2DType.PNORM, Pooling2D.Pooling2DType.MAX,};

    int cnt = 0;

    for (Pooling2D.Pooling2DType type : types) {
        log.info("Trying pooling type: [{}]", type);
        for (int m : miniBatches) {
            for (int d : depths) {
                for (int h : inHeights) {
                    for (int w : inWidths) {
                        for (int sh : strideH) {
                            for (int sw : strideW) {
                                for (int kh : sizeH) {
                                    for (int kw : sizeW) {

                                        INDArray in = Nd4j.rand(new int[]{m, d, h, w});

                                        int[] outSize = getOutputSize(in, new int[]{kh, kw}, new int[]{sh, sw}, null, true);

                                        //Calculate padding for same mode:
                                        int pHTotal = (outSize[0]-1)*sh + kh - h;
                                        int pWTotal = (outSize[1]-1)*sw + kw - w;
                                        int padTop = pHTotal / 2;
                                        int padLeft = pWTotal / 2;

                                        INDArray col = Nd4j.create(new int[]{m, d, outSize[0], outSize[1], kh, kw}, 'c');
                                        INDArray col2 = col.permute(0, 1, 4, 5, 2, 3);
                                        //INDArray col = Nd4j.createUninitialized(new int[]{m, d, kh, kw, outSize[0], outSize[1]}, 'c');
                                        //INDArray col2 = col;

                                        Convolution.im2col(in, kh, kw, sh, sw, padTop, padLeft, true, col2);

                                        INDArray col2d = col.reshape('c', m * d * outSize[0] * outSize[1], kh * kw);

                                        INDArray output = Nd4j.create(m, d, outSize[0], outSize[1]);



                                        INDArray reduced = null;
                                        switch (type) {
                                            case PNORM:
                                                int pnorm = 3;

                                                Transforms.abs(col2d, false);
                                                Transforms.pow(col2d, pnorm, false);
                                                reduced = col2d.sum(1);
                                                Transforms.pow(reduced, (1.0 / pnorm), false);

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.PNORM, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        (double) pnorm, outSize[0], outSize[1], output);

                                                break;
                                            case MAX:
                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.MAX, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.max(1);
                                                break;
                                            case AVG:

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.AVG, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.mean(1);
                                                break;
                                        }

                                        reduced = reduced.reshape('c',m,d, outSize[0], outSize[1]);

                                        assertEquals("Failed opType: " + type, reduced, output);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
 

/**
 * Element-wise differences are squared, and then summed.
 * This is modelled after the content_loss method defined in
 * https://harishnarayanan.org/writing/artistic-style-transfer/
 *
 * @param a One tensor
 * @param b Another tensor
 * @return Sum of squared errors: scalar
 */
public static double sumOfSquaredErrors(INDArray a, INDArray b) {
    INDArray diff = a.sub(b); // difference
    INDArray squares = Transforms.pow(diff, 2); // element-wise squaring
    return squares.sumNumber().doubleValue();
}
 

/**
 * Element-wise differences are squared, and then summed.
 * This is modelled after the content_loss method defined in
 * https://harishnarayanan.org/writing/artistic-style-transfer/
 *
 * @param a One tensor
 * @param b Another tensor
 * @return Sum of squared errors: scalar
 */
private double sumOfSquaredErrors(INDArray a, INDArray b) {
    INDArray diff = a.sub(b); // difference
    INDArray squares = Transforms.pow(diff, 2); // element-wise squaring
    return squares.sumNumber().doubleValue();
}