Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#mean()

public SpTree(INDArray data, Collection<INDArray> indices, String similarityFunction) {
    this.indices = indices;
    this.N = data.rows();
    this.D = data.columns();
    this.similarityFunction = similarityFunction;
    data = data.dup();
    INDArray meanY = data.mean(0);
    INDArray minY = data.min(0);
    INDArray maxY = data.max(0);
    INDArray width = Nd4j.create(data.dataType(), meanY.shape());
    for (int i = 0; i < width.length(); i++) {
        width.putScalar(i, Math.max(maxY.getDouble(i) - meanY.getDouble(i),
                meanY.getDouble(i) - minY.getDouble(i)) + Nd4j.EPS_THRESHOLD);
    }

    try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
        init(null, data, meanY, width, indices, similarityFunction);
        fill(N);
    }
}
 

public Construct4dDataSet(int nExamples, int nChannels, int height, int width) {

            INDArray allImages = Nd4j.rand(new int[] {nExamples, nChannels, height, width});
            allImages.get(NDArrayIndex.all(), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all()).muli(100)
                            .addi(200);
            allImages.get(NDArrayIndex.all(), NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.all()).muli(0.001)
                            .subi(10);

            INDArray labels = Nd4j.linspace(1, nChannels, nChannels).reshape(nChannels, 1);
            sampleDataSet = new DataSet(allImages, labels);

            expectedMean = allImages.mean(0, 2, 3);
            expectedStd = allImages.std(0, 2, 3);

            expectedLabelMean = labels.mean(0);
            expectedLabelStd = labels.std(0);

            expectedMin = allImages.min(0, 2, 3);
            expectedMax = allImages.max(0, 2, 3);
        }
 

@Test
public void testStandardize() {
    final INDArray random = Nd4j.rand(new int[]{10, 4});

    final int[] axis = new int[]{1};
    final INDArray means = random.mean(axis);
    final INDArray std = random.std(false, axis);
    final INDArray res = random.subColumnVector(means).divColumnVector(std);
    final INDArray expOut = res.norm1();

    SameDiff sd = SameDiff.create();
    SDVariable sdA = sd.var("a", random);
    SDVariable t = sd.math.standardize(sdA, axis);
    t.norm1("out");

    String err = OpValidation.validate(new TestCase(sd)
            .expectedOutput("out", expOut)
            .gradientCheck(true));
    assertNull(err, err);
}
 

private INDArray activateHelperFullArray(INDArray inputArray, int[] poolDim) {
    switch (poolingType) {
        case MAX:
            return inputArray.max(poolDim);
        case AVG:
            return inputArray.mean(poolDim);
        case SUM:
            return inputArray.sum(poolDim);
        case PNORM:
            //P norm: https://arxiv.org/pdf/1311.1780.pdf
            //out = (1/N * sum( |in| ^ p) ) ^ (1/p)
            int pnorm = layerConf().getPnorm();

            INDArray abs = Transforms.abs(inputArray, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(poolDim);

            return Transforms.pow(pNorm, 1.0 / pnorm, false);
        default:
            throw new RuntimeException("Unknown or not supported pooling type: " + poolingType + " " + layerId());
    }
}
 

@Test
public void testStandardizeNoDeviation() {
    final INDArray random = Nd4j.rand(new int[]{10, 4});
    for (int i = 0; i < 4; i++) {
        random.putScalar(1, i, 7);
    }

    final int[] axis = new int[]{1};
    final INDArray means = random.mean(axis);
    final INDArray std = random.std(false, axis);
    std.addi(std.eq(0).castTo(DataType.DOUBLE));

    final INDArray res = random.subColumnVector(means).divColumnVector(std);
    final INDArray expOut = res.norm1();

    SameDiff sd = SameDiff.create();
    SDVariable sdA = sd.var("a", random);
    SDVariable t = sd.math.standardize(sdA, axis);
    t.norm1("out");

    String err = OpValidation.validate(new TestCase(sd)
            .expectedOutput("out", expOut)
            .gradientCheck(true));
    assertNull(err, err);
}
 

@Test
public void testAnalysisBasic() throws Exception {

    RecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("iris.txt").getFile()));

    Schema s = new Schema.Builder()
            .addColumnsDouble("0", "1", "2", "3")
            .addColumnInteger("label")
            .build();

    DataAnalysis da = AnalyzeLocal.analyze(s, rr);

    System.out.println(da);

    //Compare:
    List<List<Writable>> list = new ArrayList<>();
    rr.reset();
    while(rr.hasNext()){
        list.add(rr.next());
    }

    INDArray arr = RecordConverter.toMatrix(DataType.DOUBLE, list);
    INDArray mean = arr.mean(0);
    INDArray std = arr.std(0);

    for( int i=0; i<5; i++ ){
        double m = ((NumericalColumnAnalysis)da.getColumnAnalysis().get(i)).getMean();
        double stddev = ((NumericalColumnAnalysis)da.getColumnAnalysis().get(i)).getSampleStdev();
        assertEquals(mean.getDouble(i), m, 1e-3);
        assertEquals(std.getDouble(i), stddev, 1e-3);
    }

}
 

/**
 * Calculates pca factors of a matrix, for a flags number of reduced features
 * returns the factors to scale observations 
 *
 * The return is a factor matrix to reduce (normalized) feature sets
 *
 * @see pca(INDArray, int, boolean)
 *
 * @param A the array of features, rows are results, columns are features - will be changed
 * @param nDims the number of components on which to project the features 
 * @param normalize whether to normalize (adjust each feature to have zero mean)
 * @return the reduced feature set
 */
public static INDArray pca_factor(INDArray A, int nDims, boolean normalize) {

    if (normalize) {
        // Normalize to mean 0 for each feature ( each column has 0 mean )
        INDArray mean = A.mean(0);
        A.subiRowVector(mean);
    }

    long m = A.rows();
    long n = A.columns();

    // The prepare SVD results, we'll decomp A to UxSxV'
    INDArray s = Nd4j.create(A.dataType(), m < n ? m : n);
    INDArray VT = Nd4j.create(A.dataType(), new long[]{n, n}, 'f');

    // Note - we don't care about U 
    Nd4j.getBlasWrapper().lapack().gesvd(A, s, null, VT);

    // for comparison k & nDims are the equivalent values in both methods implementing PCA

    // So now let's rip out the appropriate number of left singular vectors from
    // the V output (note we pulls rows since VT is a transpose of V)
    INDArray V = VT.transpose();
    INDArray factor = Nd4j.create(A.dataType(),new long[]{n, nDims}, 'f');
    for (int i = 0; i < nDims; i++) {
        factor.putColumn(i, V.getColumn(i));
    }

    return factor;
}
 

/**
 * Normalize data to zero mean and unit variance
 * substract by the mean and divide by the standard deviation
 *
 * @param toNormalize the ndarray to normalize
 * @return the normalized ndarray
 */
public static INDArray normalizeZeroMeanAndUnitVariance(INDArray toNormalize) {
    INDArray columnMeans = toNormalize.mean(0);
    INDArray columnStds = toNormalize.std(0);

    toNormalize.subiRowVector(columnMeans);
    //padding for non zero
    columnStds.addi(Nd4j.EPS_THRESHOLD);
    toNormalize.diviRowVector(columnStds);
    return toNormalize;
}
 

@Test
public void testMoments() {
    for (int[] axes : new int[][]{{0}, {1}, {0, 1}}) {
        INDArray input = Nd4j.linspace(1, 12, 12).reshape(3, 4);

        SameDiff sd = SameDiff.create();
        SDVariable in = sd.var("in", input);

        SDVariable[] moments = sd.math().moments(in, axes);
        INDArray expMean = input.mean(axes);
        INDArray expVar = input.var(false, axes);

        SDVariable loss;
        if (axes.length < 2) {
            loss = moments[0].add(moments[1]).std(true);
        } else {
            loss = moments[0].add(moments[1]).mean();
        }


        String msg = Arrays.toString(axes);

        TestCase tc = new TestCase(sd)
                .testName(msg)
                .expected(moments[0], expMean)
                .expected(moments[1], expVar);

        String err = OpValidation.validate(tc);
        assertNull(err);
    }
}
 

@Test
public void testColumnMean() {
    INDArray twoByThree = Nd4j.linspace(1, 4, 4, DataType.DOUBLE).reshape(2, 2);
    INDArray columnMean = twoByThree.mean(0);
    INDArray assertion = Nd4j.create(new double[] {2, 3});
    assertEquals(assertion, columnMean);
}
 

@Test
public void testRowMean() {
    INDArray twoByThree = Nd4j.linspace(1, 4, 4, DataType.DOUBLE).reshape(2, 2);
    INDArray rowMean = twoByThree.mean(1);
    INDArray assertion = Nd4j.create(new double[] {1.5, 3.5});
    assertEquals(getFailureMessage(), assertion, rowMean);


}
 

@Test
public void testMeanSumSimple() {
    System.out.println("3d");
    INDArray arr = Nd4j.ones(1, 4, 4);
    assertEquals(Nd4j.ones(1), arr.mean(1, 2));
    assertEquals(Nd4j.ones(1).muli(16), arr.sum(1, 2));

    System.out.println("4d");
    INDArray arr4 = Nd4j.ones(1, 1, 4, 4);
    INDArray arr4m = arr4.mean(2, 3);
    INDArray arr4s = arr4.sum(2, 3);
    for (int i = 0; i < arr4m.length(); i++)
        assertEquals(arr4m.getDouble(i), 1, 1e-1);
    for (int i = 0; i < arr4s.length(); i++)
        assertEquals(arr4s.getDouble(i), 16, 1e-1);

    System.out.println("5d");
    INDArray arr5 = Nd4j.ones(1, 1, 4, 4, 4);
    INDArray arr5m = arr5.mean(2, 3);
    INDArray arr5s = arr5.sum(2, 3);
    for (int i = 0; i < arr5m.length(); i++)
        assertEquals(arr5m.getDouble(i), 1, 1e-1);
    for (int i = 0; i < arr5s.length(); i++)
        assertEquals(arr5s.getDouble(i), 16, 1e-1);
    System.out.println("6d");
    INDArray arr6 = Nd4j.ones(1, 1, 4, 4, 4, 4);
    INDArray arr6Tad = arr6.javaTensorAlongDimension(0, 2, 3);
    INDArray arr6s = arr6.sum(2, 3);
    for (int i = 0; i < arr6s.length(); i++)
        assertEquals(arr6s.getDouble(i), 16, 1e-1);

    INDArray arr6m = arr6.mean(2, 3);
    for (int i = 0; i < arr6m.length(); i++)
        assertEquals(arr6m.getDouble(i), 1, 1e-1);

}
 

/**
 * Normalize data to zero mean and unit variance
 * substract by the mean and divide by the standard deviation
 *
 * @param toNormalize the ndarray to normalize
 * @return the normalized ndarray
 */
public static INDArray normalizeZeroMeanAndUnitVariance(INDArray toNormalize) {
    INDArray columnMeans = toNormalize.mean(0);
    INDArray columnStds = toNormalize.std(0);

    toNormalize.subiRowVector(columnMeans);
    //padding for non zero
    columnStds.addi(Nd4j.EPS_THRESHOLD);
    toNormalize.diviRowVector(columnStds);
    return toNormalize;
}
 

@Test
public void testLongTadOp3() {

    INDArray hugeX = Nd4j.create(2300000, 1000).assign(1.0);
    INDArray mean = hugeX.mean(1);

    for (int x = 0; x < hugeX.rows(); x++) {
        assertEquals("Failed at row " + x, 1.0, mean.getDouble(x), 1e-5);
    }
}
 

@Test
    public void testMeanSumSimple() {
//        System.out.println("3d");
        INDArray arr = Nd4j.ones(1, 4, 4);
        assertEquals(Nd4j.ones(1), arr.mean(1, 2));
        assertEquals(Nd4j.ones(1).muli(16), arr.sum(1, 2));

//        System.out.println("4d");
        INDArray arr4 = Nd4j.ones(1, 1, 4, 4);
        INDArray arr4m = arr4.mean(2, 3);
        INDArray arr4s = arr4.sum(2, 3);
        for (int i = 0; i < arr4m.length(); i++)
            assertEquals(arr4m.getDouble(i), 1, 1e-1);
        for (int i = 0; i < arr4s.length(); i++)
            assertEquals(arr4s.getDouble(i), 16, 1e-1);

//        System.out.println("5d");
        INDArray arr5 = Nd4j.ones(1, 1, 4, 4, 4);
        INDArray arr5m = arr5.mean(2, 3);
        INDArray arr5s = arr5.sum(2, 3);
        for (int i = 0; i < arr5m.length(); i++)
            assertEquals(arr5m.getDouble(i), 1, 1e-1);
        for (int i = 0; i < arr5s.length(); i++)
            assertEquals(arr5s.getDouble(i), 16, 1e-1);
//        System.out.println("6d");
        INDArray arr6 = Nd4j.ones(1, 1, 4, 4, 4, 4);
        INDArray arr6Tad = arr6.tensorAlongDimension(0, 2, 3);
        INDArray arr6s = arr6.sum(2, 3);
        for (int i = 0; i < arr6s.length(); i++)
            assertEquals(arr6s.getDouble(i), 16, 1e-1);

        INDArray arr6m = arr6.mean(2, 3);
        for (int i = 0; i < arr6m.length(); i++)
            assertEquals(arr6m.getDouble(i), 1, 1e-1);

    }
 

@Ignore
@Test
public void testScale() {
    StandardScaler scaler = new StandardScaler();
    DataSetIterator iter = new IrisDataSetIterator(10, 150);
    scaler.fit(iter);
    INDArray featureMatrix = new IrisDataSetIterator(150, 150).next().getFeatureMatrix();
    INDArray mean = featureMatrix.mean(0);
    INDArray std = featureMatrix.std(0);
    System.out.println(mean);
}
 

@Test
    public void testDnnForwardPass() {
        int nOut = 10;
        Layer l = getLayer(nOut, 0.0, false, -1, -1);
        assertEquals(4 * nOut, l.numParams()); //Gamma, beta, global mean, global var

        INDArray randInput = Nd4j.rand(100, nOut);
        INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces());

        INDArray mean = output.mean(0);
        INDArray stdev = output.std(false, 0);

//        System.out.println(Arrays.toString(mean.data().asFloat()));

        assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f);
        assertEquals(Nd4j.ones(nOut), stdev);

        //If we fix gamma/beta: expect different mean and variance...
        double gamma = 2.0;
        double beta = 3.0;
        l = getLayer(nOut, 0.0, true, gamma, beta);
        assertEquals(2 * nOut, l.numParams()); //Should have only global mean/var parameters
        output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces());
        mean = output.mean(0);
        stdev = output.std(false, 0);

        assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean);
        assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev);
    }
 

/**
 * Calculates pca vectors of a matrix, for a given variance. A larger variance (99%)
 * will result in a higher order feature set.
 *
 * To use the returned factor: multiply feature(s) by the factor to get a reduced dimension
 *
 * INDArray Areduced = A.mmul( factor ) ;
 * 
 * The array Areduced is a projection of A onto principal components
 *
 * @see pca(INDArray, double, boolean)
 *
 * @param A the array of features, rows are results, columns are features - will be changed
 * @param variance the amount of variance to preserve as a float 0 - 1
 * @param normalize whether to normalize (set features to have zero mean)
 * @return the matrix to mulitiply a feature by to get a reduced feature set
 */
public static INDArray pca_factor(INDArray A, double variance, boolean normalize) {
    if (normalize) {
        // Normalize to mean 0 for each feature ( each column has 0 mean )
        INDArray mean = A.mean(0);
        A.subiRowVector(mean);
    }

    long m = A.rows();
    long n = A.columns();

    // The prepare SVD results, we'll decomp A to UxSxV'
    INDArray s = Nd4j.create(A.dataType(), m < n ? m : n);
    INDArray VT = Nd4j.create(A.dataType(), new long[]{n, n}, 'f');

    // Note - we don't care about U 
    Nd4j.getBlasWrapper().lapack().gesvd(A, s, null, VT);

    // Now convert the eigs of X into the eigs of the covariance matrix
    for (int i = 0; i < s.length(); i++) {
        s.putScalar(i, Math.sqrt(s.getDouble(i)) / (m - 1));
    }

    // Now find how many features we need to preserve the required variance
    // Which is the same percentage as a cumulative sum of the eigenvalues' percentages
    double totalEigSum = s.sumNumber().doubleValue() * variance;
    int k = -1; // we will reduce to k dimensions
    double runningTotal = 0;
    for (int i = 0; i < s.length(); i++) {
        runningTotal += s.getDouble(i);
        if (runningTotal >= totalEigSum) { // OK I know it's a float, but what else can we do ?
            k = i + 1; // we will keep this many features to preserve the reqd. variance
            break;
        }
    }
    if (k == -1) { // if we need everything
        throw new RuntimeException("No reduction possible for reqd. variance - use smaller variance");
    }
    // So now let's rip out the appropriate number of left singular vectors from
    // the V output (note we pulls rows since VT is a transpose of V)
    INDArray V = VT.transpose();
    INDArray factor = Nd4j.createUninitialized(A.dataType(), new long[]{n, k}, 'f');
    for (int i = 0; i < k; i++) {
        factor.putColumn(i, V.getColumn(i));
    }

    return factor;
}
 

/**
 * Add rows of data to the statistics
 *
 * @param data the matrix containing multiple rows of data to include
 * @param mask (optionally) the mask of the data, useful for e.g. time series
 */
public Builder add(@NonNull INDArray data, INDArray mask) {
    data = DataSetUtil.tailor2d(data, mask);

    // Using https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm
    if (data == null) {
        // Nothing to add. Either data is empty or completely masked. Just skip it, otherwise we will get
        // null pointer exceptions.
        return this;
    }
    INDArray mean = data.mean(0);
    INDArray variance = data.var(false, 0);
    long count = data.size(0);

    if (runningMean == null) {
        // First batch
        runningMean = mean;
        runningVariance = variance;
        runningCount = count;

        if (data.size(0) == 1) {
            //Handle edge case: currently, reduction ops may return the same array
            //But we don't want to modify this array in-place later
            runningMean = runningMean.dup();
            runningVariance = runningVariance.dup();
        }
    } else {
        // Update running variance
        INDArray deltaSquared = Transforms.pow(mean.subRowVector(runningMean), 2);
        INDArray mB = variance.muli(count);
        runningVariance.muli(runningCount).addiRowVector(mB)
                        .addiRowVector(deltaSquared
                                        .muli((float) (runningCount * count) / (runningCount + count)))
                        .divi(runningCount + count);

        // Update running count
        runningCount += count;

        // Update running mean
        INDArray xMinusMean = data.subRowVector(runningMean);
        runningMean.addi(xMinusMean.sum(0).divi(runningCount));
    }

    return this;
}
 

/**
 * This method returns mean vector, built from words/labels passed in
 *
 * @param labels
 * @return
 */
@Override
public INDArray getWordVectorsMean(Collection<String> labels) {
    INDArray array = getWordVectors(labels);
    return array.mean(0);
}