Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#getColumn()

public boolean compareColumn(INDArray newData) {
    float[] newMap = measureState(newData);

    if (newMap.length != map.length) {
        System.out.println("Different map lengths");
        return false;
    }

    if (Arrays.equals(map, newMap)) {
        System.out.println("Maps are equal");
        return false;
    }

    for (int x = 0; x < newData.rows(); x++) {
        INDArray column = newData.getColumn(x);
        double val = column.getDouble(0);
        for (int y = 0; y < column.lengthLong(); y++ ) {
            if (Math.abs(column.getFloat(y) - val) > Nd4j.EPS_THRESHOLD) {
                System.out.print("Different data in a column: " + column.getFloat(y));
                return false;
            }
        }
    }

    return true;
}
 

public boolean compareColumn(INDArray newData) {
    float[] newMap = measureState(newData);

    if (newMap.length != map.length) {
        System.out.println("Different map lengths");
        return false;
    }

    if (Arrays.equals(map, newMap)) {
        System.out.println("Maps are equal");
        return false;
    }

    for (int x = 0; x < newData.rows(); x++) {
        INDArray column = newData.getColumn(x);
        double val = column.getDouble(0);
        for (int y = 0; y < column.lengthLong(); y++) {
            if (Math.abs(column.getFloat(y) - val) > Nd4j.EPS_THRESHOLD) {
                System.out.print("Different data in a column: " + column.getFloat(y));
                return false;
            }
        }
    }

    return true;
}
 

@Test
public void testViewDetach_1() {
    WorkspaceConfiguration configuration = WorkspaceConfiguration.builder().initialSize(10000000).overallocationLimit(3.0)
            .policyAllocation(AllocationPolicy.OVERALLOCATE).policySpill(SpillPolicy.REALLOCATE)
            .policyLearning(LearningPolicy.FIRST_LOOP).policyReset(ResetPolicy.BLOCK_LEFT).build();

    Nd4jWorkspace workspace =
            (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(configuration, "WS109");

    INDArray row = Nd4j.linspace(1, 10, 10);
    INDArray exp = Nd4j.create(10).assign(2.0);
    INDArray result = null;
    try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS109")) {
        INDArray matrix = Nd4j.create(10, 10);
        for (int e = 0; e < matrix.rows(); e++)
            matrix.getRow(e).assign(row);


        INDArray column = matrix.getColumn(1);
        assertTrue(column.isView());
        assertTrue(column.isAttached());
        result = column.detach();
    }

    assertFalse(result.isView());
    assertFalse(result.isAttached());
    assertEquals(exp, result);
}
 

@Test
public void testScatterMin() {
    NDBase base = new NDBase();

    //from testScatterOpGradients.
    INDArray x = Nd4j.ones(DataType.DOUBLE, 20, 10).add(1.0);
    INDArray indices = Nd4j.createFromArray(3, 4, 5, 10, 18);
    INDArray updates = Nd4j.ones(DataType.DOUBLE, 5, 10).add(1.0);
    INDArray y = base.scatterMin(x,indices, updates);

    y = y.getColumn(0);
    INDArray  y_exp = Nd4j.createFromArray(2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0);
    assertEquals(y_exp, y);
}
 

@Test
public void testScatterDiv() {
    NDBase base = new NDBase();

    //from testScatterOpGradients.
    INDArray x = Nd4j.ones(DataType.DOUBLE, 20, 10).add(1.0);
    INDArray indices = Nd4j.createFromArray(3, 4, 5, 10, 18);
    INDArray updates = Nd4j.ones(DataType.DOUBLE, 5, 10).add(1.0);
    INDArray y = base.scatterDiv(x,indices, updates);

    y = y.getColumn(0);
    INDArray  y_exp = Nd4j.createFromArray(2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 2.0);
    assertEquals(y_exp, y);
}
 

@Test
public void testScatterAdd() {
    NDBase base = new NDBase();

    //from testScatterOpGradients.
    INDArray x = Nd4j.ones(DataType.DOUBLE, 20, 10);
    INDArray indices = Nd4j.createFromArray(3, 4, 5, 10, 18);
    INDArray updates = Nd4j.ones(DataType.DOUBLE, 5, 10);
    INDArray y = base.scatterAdd(x,indices, updates);

    y = y.getColumn(0);
    INDArray  y_exp = Nd4j.createFromArray(1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0);
    assertEquals(y_exp, y);
}
 

@Test
public void testGetColumnEdgeCase() {
    INDArray colVec = Nd4j.linspace(1, 5, 5, DataType.DOUBLE).reshape(1, -1).transpose();
    INDArray get = colVec.getColumn(0); //Returning shape [1,1]

    assertArrayEquals(new long[] {5, 1}, get.shape());
    assertEquals(colVec, get);
}
 

@Override
public INDArray getPFactor(int M, INDArray ipiv) {
    // The simplest permutation is the identity matrix
    INDArray P = Nd4j.eye(M); // result is a square matrix with given size
    for (int i = 0; i < ipiv.length(); i++) {
        int pivot = ipiv.getInt(i) - 1; // Did we swap row #i with anything?
        if (pivot > i) { // don't reswap when we get lower down in the vector
            INDArray v1 = P.getColumn(i).dup(); // because of row vs col major order we'll ...
            INDArray v2 = P.getColumn(pivot); // ... make a transposed matrix immediately
            P.putColumn(i, v2);
            P.putColumn(pivot, v1); // note dup() above is required - getColumn() is a 'view'
        }
    }
    return P; // the permutation matrix - contains a single 1 in any row and column
}
 

@Test
    public void testShuffleMeta() {
        int nExamples = 20;
        int nColumns = 4;

        INDArray f = Nd4j.zeros(nExamples, nColumns);
        INDArray l = Nd4j.zeros(nExamples, nColumns);
        List<Integer> meta = new ArrayList<>();

        for (int i = 0; i < nExamples; i++) {
            f.getRow(i).assign(i);
            l.getRow(i).assign(i);
            meta.add(i);
        }

        DataSet ds = new DataSet(f, l);
        ds.setExampleMetaData(meta);

        for (int i = 0; i < 10; i++) {
            ds.shuffle();
            INDArray fCol = f.getColumn(0);
            INDArray lCol = l.getColumn(0);
//            System.out.println(fCol + "\t" + ds.getExampleMetaData());
            for (int j = 0; j < nExamples; j++) {
                int fVal = (int) fCol.getDouble(j);
                int lVal = (int) lCol.getDouble(j);
                int metaVal = (Integer) ds.getExampleMetaData().get(j);

                assertEquals(fVal, lVal);
                assertEquals(fVal, metaVal);
            }
        }
    }
 

@Test
    public void testGetColumnFortran() {
        INDArray n = Nd4j.create(Nd4j.linspace(1, 4, 4, DataType.DOUBLE).data(), new long[] {2, 2});
        INDArray column = Nd4j.create(new double[] {1, 2});
        INDArray column2 = Nd4j.create(new double[] {3, 4});
        INDArray testColumn = n.getColumn(0);
        INDArray testColumn1 = n.getColumn(1);
//        log.info("testColumn shape: {}", Arrays.toString(testColumn.shapeInfoDataBuffer().asInt()));
        assertEquals(column, testColumn);
        assertEquals(column2, testColumn1);

    }
 

@Override
public INDArray ndOp(INDArray features, INDArray adjacencyMatrix) {
    double sigma = 16;
    INDArray[] sumsOfSquareDiffs = new INDArray[adjacencyMatrix.rows()];
    for (int node = 0; node < adjacencyMatrix.rows(); node++) {
        INDArray column = adjacencyMatrix.getColumn(node);
        INDArray repeat = features.getRow(node).repeat(0, features.rows()).muliColumnVector(column);
        INDArray sub = repeat.sub(features.mulColumnVector(column));
        sumsOfSquareDiffs[node] = Transforms.pow(sub, 2).sum(0);
    }
    INDArray sumOfSquareDiffs = Nd4j.vstack(sumsOfSquareDiffs).muli(-(1d / Math.pow(sigma, 2)));
    return Transforms.exp(sumOfSquareDiffs);
}
 

@Override
public INDArray getPFactor(int M, INDArray ipiv) {
    // The simplest permutation is the identity matrix
    INDArray P = Nd4j.eye(M); // result is a square matrix with given size
    for (int i = 0; i < ipiv.length(); i++) {
        int pivot = ipiv.getInt(i) - 1; // Did we swap row #i with anything?
        if (pivot > i) { // don't reswap when we get lower down in the vector
            INDArray v1 = P.getColumn(i).dup(); // because of row vs col major order we'll ...
            INDArray v2 = P.getColumn(pivot); // ... make a transposed matrix immediately
            P.putColumn(i, v2);
            P.putColumn(pivot, v1); // note dup() above is required - getColumn() is a 'view'
        }
    }
    return P; // the permutation matrix - contains a single 1 in any row and column
}
 

@Test
public void testViewDetach_1() throws Exception {
    WorkspaceConfiguration configuration = WorkspaceConfiguration.builder().initialSize(10000000).overallocationLimit(3.0)
            .policyAllocation(AllocationPolicy.OVERALLOCATE).policySpill(SpillPolicy.REALLOCATE)
            .policyLearning(LearningPolicy.FIRST_LOOP).policyReset(ResetPolicy.BLOCK_LEFT).build();

    Nd4jWorkspace workspace =
            (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(configuration, "WS109");

    INDArray row = Nd4j.linspace(1, 10, 10);
    INDArray exp = Nd4j.create(1, 10).assign(2.0);
    INDArray result = null;
    try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS109")) {
        INDArray matrix = Nd4j.create(10, 10);
        for (int e = 0; e < matrix.rows(); e++)
            matrix.getRow(e).assign(row);


        INDArray column = matrix.getColumn(1);
        assertTrue(column.isView());
        assertTrue(column.isAttached());
        result = column.detach();
    }

    assertFalse(result.isView());
    assertFalse(result.isAttached());
    assertEquals(exp, result);
}
 

@Test
public void testScatterSub() {
    NDBase base = new NDBase();

    //from testScatterOpGradients.
    INDArray x = Nd4j.ones(DataType.DOUBLE, 20, 10).add(1.0);
    INDArray indices = Nd4j.createFromArray(3, 4, 5, 10, 18);
    INDArray updates = Nd4j.ones(DataType.DOUBLE, 5, 10).add(1.0);
    INDArray y = base.scatterSub(x,indices, updates);

    y = y.getColumn(0);
    INDArray  y_exp = Nd4j.createFromArray(2.0, 2.0, 2.0, 0.0, 0.0, 0.0, 2.0, 2.0, 2.0, 2.0, 0.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 0.0, 2.0);
    assertEquals(y_exp, y);
}
 

@Test
public void testShuffleMeta() {
    int nExamples = 20;
    int nColumns = 4;

    INDArray f = Nd4j.zeros(nExamples, nColumns);
    INDArray l = Nd4j.zeros(nExamples, nColumns);
    List<Integer> meta = new ArrayList<>();

    for (int i = 0; i < nExamples; i++) {
        f.getRow(i).assign(i);
        l.getRow(i).assign(i);
        meta.add(i);
    }

    DataSet ds = new DataSet(f, l);
    ds.setExampleMetaData(meta);

    for (int i = 0; i < 10; i++) {
        ds.shuffle();
        INDArray fCol = f.getColumn(0);
        INDArray lCol = l.getColumn(0);
        System.out.println(fCol + "\t" + ds.getExampleMetaData());
        for (int j = 0; j < nExamples; j++) {
            int fVal = (int) fCol.getDouble(j);
            int lVal = (int) lCol.getDouble(j);
            int metaVal = (Integer) ds.getExampleMetaData().get(j);

            assertEquals(fVal, lVal);
            assertEquals(fVal, metaVal);
        }
    }
}
 

@Test
public void testRemoveInnerLoopForComparisons() {
    final double[][] doubles = {
            {0, 1, 1, 1},
            {0, 1, 2, 2},
            {0, 1, 3, 2},
            {0, 2, 3, 3},
    };

    final INDArray indArray = Nd4j.create(doubles);
    final double lambda = 0.6;

    final INDArray[] scores = new INDArray[indArray.rows()];
    for (int i = 0; i < indArray.columns(); i++) {
        final INDArray column = indArray.getColumn(i);
        final INDArray zerosToSum = indArray.subColumnVector(column);
        scores[i] = zerosToSum.condi(Conditions.equals(0)).sum(0).divi(indArray.rows());
    }
    final INDArray indScores = Nd4j.vstack(scores);
    System.out.println("scores = \n" + indScores);

    indScores.condi(Conditions.greaterThan(lambda));
    System.out.println("adjacency matrix with self-loops = \n" + indScores);

    // this line should be optional - not sure better with or without
    Nd4j.doAlongDiagonal(indScores, input -> 0);

    System.out.println("adjacency matrix with self loops removed = \n" + indScores);
}
 

@Test
public void testEvaluationBinary() {
    //Compare EvaluationBinary to Evaluation class
    DataType dtypeBefore = Nd4j.defaultFloatingPointType();
    EvaluationBinary first = null;
    String sFirst = null;
    try {
        for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.INT}) {
            Nd4j.setDefaultDataTypes(globalDtype, globalDtype.isFPType() ? globalDtype : DataType.DOUBLE);
            for (DataType lpDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {

                Nd4j.getRandom().setSeed(12345);

                int nExamples = 50;
                int nOut = 4;
                long[] shape = {nExamples, nOut};

                INDArray labels = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(lpDtype, shape), 0.5));

                INDArray predicted = Nd4j.rand(lpDtype, shape);
                INDArray binaryPredicted = predicted.gt(0.5);

                EvaluationBinary eb = new EvaluationBinary();
                eb.eval(labels, predicted);

                //System.out.println(eb.stats());

                double eps = 1e-6;
                for (int i = 0; i < nOut; i++) {
                    INDArray lCol = labels.getColumn(i,true);
                    INDArray pCol = predicted.getColumn(i,true);
                    INDArray bpCol = binaryPredicted.getColumn(i,true);

                    int countCorrect = 0;
                    int tpCount = 0;
                    int tnCount = 0;
                    for (int j = 0; j < lCol.length(); j++) {
                        if (lCol.getDouble(j) == bpCol.getDouble(j)) {
                            countCorrect++;
                            if (lCol.getDouble(j) == 1) {
                                tpCount++;
                            } else {
                                tnCount++;
                            }
                        }
                    }
                    double acc = countCorrect / (double) lCol.length();

                    Evaluation e = new Evaluation();
                    e.eval(lCol, pCol);

                    assertEquals(acc, eb.accuracy(i), eps);
                    assertEquals(e.accuracy(), eb.scoreForMetric(ACCURACY, i), eps);
                    assertEquals(e.precision(1), eb.scoreForMetric(PRECISION, i), eps);
                    assertEquals(e.recall(1), eb.scoreForMetric(RECALL, i), eps);
                    assertEquals(e.f1(1), eb.scoreForMetric(F1, i), eps);
                    assertEquals(e.falseAlarmRate(), eb.scoreForMetric(FAR, i), eps);
                    assertEquals(e.falsePositiveRate(1), eb.falsePositiveRate(i), eps);


                    assertEquals(tpCount, eb.truePositives(i));
                    assertEquals(tnCount, eb.trueNegatives(i));

                    assertEquals((int) e.truePositives().get(1), eb.truePositives(i));
                    assertEquals((int) e.trueNegatives().get(1), eb.trueNegatives(i));
                    assertEquals((int) e.falsePositives().get(1), eb.falsePositives(i));
                    assertEquals((int) e.falseNegatives().get(1), eb.falseNegatives(i));

                    assertEquals(nExamples, eb.totalCount(i));

                    String s = eb.stats();
                    if(first == null) {
                        first = eb;
                        sFirst = s;
                    } else {
                        assertEquals(first, eb);
                        assertEquals(sFirst, s);
                    }
                }
            }
        }
    } finally {
        Nd4j.setDefaultDataTypes(dtypeBefore, dtypeBefore);
    }
}
 

private double[] computeScoreNumDenom(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                boolean average) {
    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    long n = labels.size(1);
    if (n != 1 && n != 2) {
        throw new UnsupportedOperationException(
                        "For binary classification: expect output size of 1 or 2. Got: " + n);
    }

    //First: determine positives and negatives
    INDArray isPositiveLabel;
    INDArray isNegativeLabel;
    INDArray pClass0;
    INDArray pClass1;
    if (n == 1) {
        isPositiveLabel = labels;
        isNegativeLabel = isPositiveLabel.rsub(1.0);
        pClass0 = output.rsub(1.0);
        pClass1 = output;
    } else {
        isPositiveLabel = labels.getColumn(1);
        isNegativeLabel = labels.getColumn(0);
        pClass0 = output.getColumn(0);
        pClass1 = output.getColumn(1);
    }

    if (mask != null) {
        isPositiveLabel = isPositiveLabel.mulColumnVector(mask);
        isNegativeLabel = isNegativeLabel.mulColumnVector(mask);
    }

    double tp = isPositiveLabel.mul(pClass1).sumNumber().doubleValue();
    double fp = isNegativeLabel.mul(pClass1).sumNumber().doubleValue();
    double fn = isPositiveLabel.mul(pClass0).sumNumber().doubleValue();

    double numerator = (1.0 + beta * beta) * tp;
    double denominator = (1.0 + beta * beta) * tp + beta * beta * fn + fp;

    return new double[] {numerator, denominator};
}
 

@Override
public void eval(INDArray labels, INDArray predictions, INDArray mask, List<? extends Serializable> recordMetaData) {
    Triple<INDArray,INDArray, INDArray> p = BaseEvaluation.reshapeAndExtractNotMasked(labels, predictions, mask, axis);
    INDArray labels2d = p.getFirst();
    INDArray predictions2d = p.getSecond();
    INDArray maskArray = p.getThird();

    if (underlying != null && underlying.length != labels2d.size(1)) {
        throw new IllegalStateException("Labels array does not match stored state size. Expected labels array with "
                        + "size " + underlying.length + ", got labels array with size " + labels2d.size(1));
    }

    if (labels2d.rank() == 3) {
        evalTimeSeries(labels2d, predictions2d, maskArray);
        return;
    }

    if(labels2d.dataType() != predictions2d.dataType())
        labels2d = labels2d.castTo(predictions2d.dataType());

    int n = (int) labels2d.size(1);
    if (underlying == null) {
        underlying = new ROC[n];
        for (int i = 0; i < n; i++) {
            underlying[i] = new ROC(thresholdSteps, rocRemoveRedundantPts);
        }
    }

    int[] perExampleNonMaskedIdxs = null;
    for (int i = 0; i < n; i++) {
        INDArray prob = predictions2d.getColumn(i).reshape(predictions2d.size(0), 1);
        INDArray label = labels2d.getColumn(i).reshape(labels2d.size(0), 1);
        if (maskArray != null) {
            //If mask array is present, pull out the non-masked rows only
            INDArray m;
            boolean perExampleMasking = false;
            if (maskArray.isColumnVectorOrScalar()) {
                //Per-example masking
                m = maskArray;
                perExampleMasking = true;
            } else {
                //Per-output masking
                m = maskArray.getColumn(i);
            }
            int[] rowsToPull;

            if (perExampleNonMaskedIdxs != null) {
                //Reuse, per-example masking
                rowsToPull = perExampleNonMaskedIdxs;
            } else {
                int nonMaskedCount = m.sumNumber().intValue();
                rowsToPull = new int[nonMaskedCount];
                val maskSize = m.size(0);
                int used = 0;
                for (int j = 0; j < maskSize; j++) {
                    if (m.getDouble(j) != 0.0) {
                        rowsToPull[used++] = j;
                    }
                }
                if (perExampleMasking) {
                    perExampleNonMaskedIdxs = rowsToPull;
                }
            }

            //TODO Temporary workaround for: https://github.com/deeplearning4j/deeplearning4j/issues/7102
            if(prob.isView())
                prob = prob.dup();
            if(label.isView())
                label = label.dup();

            prob = Nd4j.pullRows(prob, 1, rowsToPull); //1: tensor along dim 1
            label = Nd4j.pullRows(label, 1, rowsToPull);
        }

        underlying[i].eval(label, prob);
    }
}
 

@Test
public void testColumns() {
    INDArray arr = Nd4j.create(new long[] {3, 2}).castTo(DataType.DOUBLE);
    INDArray column = Nd4j.create(new double[] {1, 2, 3});
    arr.putColumn(0, column);

    INDArray firstColumn = arr.getColumn(0);

    assertEquals(column, firstColumn);


    INDArray column1 = Nd4j.create(new double[] {4, 5, 6});
    arr.putColumn(1, column1);
    INDArray testRow1 = arr.getColumn(1);
    assertEquals(column1, testRow1);


    INDArray evenArr = Nd4j.create(new double[] {1, 2, 3, 4}, new long[] {2, 2});
    INDArray put = Nd4j.create(new double[] {5, 6});
    evenArr.putColumn(1, put);
    INDArray testColumn = evenArr.getColumn(1);
    assertEquals(put, testColumn);


    INDArray n = Nd4j.create(Nd4j.linspace(1, 4, 4, DataType.DOUBLE).data(), new long[] {2, 2}).castTo(DataType.DOUBLE);
    INDArray column23 = n.getColumn(0);
    INDArray column12 = Nd4j.create(new double[] {1, 2});
    assertEquals(column23, column12);


    INDArray column0 = n.getColumn(1);
    INDArray column01 = Nd4j.create(new double[] {3, 4});
    assertEquals(column0, column01);


}