Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#rows()

public static List<DataSet> loadIris(int from, int to) throws IOException {
    ClassPathResource resource = new ClassPathResource("/iris.dat", IrisUtils.class.getClassLoader());
    @SuppressWarnings("unchecked")
    List<String> lines = IOUtils.readLines(resource.getInputStream());
    List<DataSet> list = new ArrayList<>();
    INDArray ret = Nd4j.ones(Math.abs(to - from), 4);
    double[][] outcomes = new double[lines.size()][3];
    int putCount = 0;

    for (int i = from; i < to; i++) {
        String line = lines.get(i);
        String[] split = line.split(",");

        addRow(ret, putCount++, split);

        String outcome = split[split.length - 1];
        double[] rowOutcome = new double[3];
        rowOutcome[Integer.parseInt(outcome)] = 1;
        outcomes[i] = rowOutcome;
    }

    for (int i = 0; i < ret.rows(); i++)
        list.add(new DataSet(ret.getRow(i), Nd4j.create(outcomes[from + i])));

    return list;
}
 

@Test
public void testNadam() {
    int rows = 10;
    int cols = 2;

    NadamUpdater grad = new NadamUpdater(new Nadam());
    grad.setStateViewArray(Nd4j.zeros(1, 2 * rows * cols), new long[] {rows, cols}, 'c', true);
    INDArray W = Nd4j.zeros(rows, cols);
    Distribution dist = Nd4j.getDistributions().createNormal(1e-3, 1e-3);
    for (int i = 0; i < W.rows(); i++)
        W.putRow(i, Nd4j.create(dist.sample(W.columns())));

    for (int i = 0; i < 5; i++) {
        //            String learningRates = String.valueOf("\nAdamUpdater\n " + grad.applyUpdater(W, i)).replaceAll(";", "\n");
        //            System.out.println(learningRates);
        W.addi(Nd4j.randn(rows, cols));
    }
}
 

public boolean compareColumn(INDArray newData) {
            float[] newMap = measureState(newData);

            if (newMap.length != map.length) {
                System.out.println("Different map lengths");
                return false;
            }

            if (Arrays.equals(map, newMap)) {
//                System.out.println("Maps are equal");
                return false;
            }

            for (int x = 0; x < newData.rows(); x++) {
                INDArray column = newData.getColumn(x);
                double val = column.getDouble(0);
                for (int y = 0; y < column.length(); y++) {
                    if (Math.abs(column.getFloat(y) - val) > Nd4j.EPS_THRESHOLD) {
                        System.out.print("Different data in a column: " + column.getFloat(y));
                        return false;
                    }
                }
            }

            return true;
        }
 

/**
 * Get top N elements
 *
 * @param vec the vec to extract the top elements from
 * @param N the number of elements to extract
 * @return the indices and the sorted top N elements
 */
private List<Double> getTopN(INDArray vec, int N) {
    BasicModelUtils.ArrayComparator comparator = new BasicModelUtils.ArrayComparator();
    PriorityQueue<Double[]> queue = new PriorityQueue<>(vec.rows(), comparator);

    for (int j = 0; j < vec.length(); j++) {
        final Double[] pair = new Double[] {vec.getDouble(j), (double) j};
        if (queue.size() < N) {
            queue.add(pair);
        } else {
            Double[] head = queue.peek();
            if (comparator.compare(pair, head) > 0) {
                queue.poll();
                queue.add(pair);
            }
        }
    }

    List<Double> lowToHighSimLst = new ArrayList<>();

    while (!queue.isEmpty()) {
        double ind = queue.poll()[1];
        lowToHighSimLst.add(ind);
    }
    return Lists.reverse(lowToHighSimLst);
}
 

@Override
public INDArray getLFactor(INDArray A) {
    // FIXME: int cast
    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    INDArray L = Nd4j.create(m, n);
    for (int r = 0; r < m; r++) {
        for (int c = 0; c < n; c++) {
            if (r > c && r < m && c < n) {
                L.putScalar(r, c, A.getFloat(r, c));
            } else if (r < c) {
                L.putScalar(r, c, 0.f);
            } else {
                L.putScalar(r, c, 1.f);
            }
        }
    }
    return L;
}
 

@Test
public void testNesterovs() {
    int rows = 10;
    int cols = 2;


    NesterovsUpdater grad = new NesterovsUpdater(new Nesterovs(0.5, 0.9));
    grad.setStateViewArray(Nd4j.zeros(1, rows * cols), new long[] {rows, cols}, 'c', true);
    INDArray W = Nd4j.zeros(rows, cols);
    Distribution dist = Nd4j.getDistributions().createNormal(1, 1);
    for (int i = 0; i < W.rows(); i++)
        W.putRow(i, Nd4j.create(dist.sample(W.columns())));

    for (int i = 0; i < 5; i++) {
        //            String learningRates = String.valueOf("\nAdagrad\n " + grad.applyUpdater(W, i)).replaceAll(";", "\n");
        //            System.out.println(learningRates);
        W.addi(Nd4j.randn(rows, cols));
    }
}
 

@Test
public void testBroadCast() {
    INDArray n = Nd4j.linspace(1, 4, 4, DataType.DOUBLE);
    INDArray broadCasted = n.broadcast(5, 4);
    for (int i = 0; i < broadCasted.rows(); i++) {
        assertEquals(n, broadCasted.getRow(i));
    }

    INDArray broadCast2 = broadCasted.getRow(0).broadcast(5, 4);
    assertEquals(broadCasted, broadCast2);


    INDArray columnBroadcast = n.reshape(4,1).broadcast(4, 5);
    for (int i = 0; i < columnBroadcast.columns(); i++) {
        assertEquals(columnBroadcast.getColumn(i), n.reshape(4));
    }

    INDArray fourD = Nd4j.create(1, 2, 1, 1);
    INDArray broadCasted3 = fourD.broadcast(1, 2, 36, 36);
    assertTrue(Arrays.equals(new long[] {1, 2, 36, 36}, broadCasted3.shape()));
}
 

@Test
public void testRemoveInnerLoopForComparisons() {
    final double[][] doubles = {
            {0, 1, 1, 1},
            {0, 1, 2, 2},
            {0, 1, 3, 2},
            {0, 2, 3, 3},
    };

    final INDArray indArray = Nd4j.create(doubles);
    final double lambda = 0.6;

    final INDArray[] scores = new INDArray[indArray.rows()];
    for (int i = 0; i < indArray.columns(); i++) {
        final INDArray column = indArray.getColumn(i);
        final INDArray zerosToSum = indArray.subColumnVector(column);
        scores[i] = zerosToSum.condi(Conditions.equals(0)).sum(0).divi(indArray.rows());
    }
    final INDArray indScores = Nd4j.vstack(scores);
    System.out.println("scores = \n" + indScores);

    indScores.condi(Conditions.greaterThan(lambda));
    System.out.println("adjacency matrix with self-loops = \n" + indScores);

    // this line should be optional - not sure better with or without
    Nd4j.doAlongDiagonal(indScores, input -> 0);

    System.out.println("adjacency matrix with self loops removed = \n" + indScores);
}
 

/**
 * Creates an out come vector from the specified inputs
 *
 * @param index       the index of the label
 * @param numOutcomes the number of possible outcomes
 * @return a binary label matrix used for supervised learning
 */
public static INDArray toOutcomeMatrix(int[] index, long numOutcomes) {
    INDArray ret = Nd4j.create(index.length, numOutcomes);
    for (int i = 0; i < ret.rows(); i++) {
        int[] nums = new int[(int) numOutcomes];
        nums[index[i]] = 1;
        ret.putRow(i, NDArrayUtil.toNDArray(nums));
    }

    return ret;
}
 

@Test
public void testGpuBlas8akaMultipleThreadsMultipleDevicesSameData() throws Exception {

    Random rnd = new Random(42);

    INDArray array1 = Nd4j.create(100, 757);
    INDArray array2 = Nd4j.create(100, 757);
    for (int y = 0; y < array1.rows(); y++) {
        float[] srcArray1 = new float[757];
        float[] srcArray2 = new float[757];

        for (int x = 0; x < srcArray1.length; x++) {
            srcArray1[x] = rnd.nextFloat();
            srcArray2[x] = rnd.nextFloat();
        }

        array1.putRow(y, Nd4j.create(srcArray1));
        array2.putRow(y, Nd4j.create(srcArray2));
    }

    GpuThreadExternalData[] threads = new GpuThreadExternalData[4];
    for (int x =0; x< threads.length; x++) {
        GpuThreadExternalData thread = new GpuThreadExternalData(array1, array2);
        thread.start();
        threads[x] = thread;
    }

    for (int x = 0; x < threads.length; x++) {
        threads[x].join();
    }

    log.info("Finished");
}
 

public float[] measureState(INDArray data) {
    float[] result = new float[data.rows()];

    for (int x = 0; x < data.rows(); x++) {
        result[x] = data.getRow(x).getFloat(0);
    }

    return result;
}
 

/**
 * Convert a set of records in to a matrix
 * @param matrix the records ot convert
 * @return the matrix for the records
 */
public static List<List<Writable>> toRecords(INDArray matrix) {
    List<List<Writable>> ret = new ArrayList<>();
    for (int i = 0; i < matrix.rows(); i++) {
        ret.add(RecordConverter.toRecord(matrix.getRow(i)));
    }

    return ret;
}
 

/**
 *
 * @param matrix
 * @return
 */
public static List<Point> toPoints(INDArray matrix) {
    List<Point> arr = new ArrayList<>(matrix.rows());
    for (int i = 0; i < matrix.rows(); i++) {
        arr.add(new Point(matrix.getRow(i)));
    }

    return arr;
}
 

@Override
public void gesvd(INDArray A, INDArray S, INDArray U, INDArray VT) {
    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    byte jobu = (byte) (U == null ? 'N' : 'A');
    byte jobvt = (byte) (VT == null ? 'N' : 'A');

    INDArray INFO = Nd4j.createArrayFromShapeBuffer(Nd4j.getDataBufferFactory().createInt(1),
                    Nd4j.getShapeInfoProvider().createShapeInformation(new long[] {1, 1}, DataType.INT).getFirst());

    if (A.data().dataType() == DataType.DOUBLE)
        dgesvd(jobu, jobvt, m, n, A, S, U, VT, INFO);
    else if (A.data().dataType() == DataType.FLOAT)
        sgesvd(jobu, jobvt, m, n, A, S, U, VT, INFO);
    else
        throw new UnsupportedOperationException();

    if (INFO.getInt(0) < 0) {
        throw new Error("Parameter #" + INFO.getInt(0) + " to gesvd() was not valid");
    } else if (INFO.getInt(0) > 0) {
        log.warn("The matrix contains singular elements. Check S matrix at row " + INFO.getInt(0));
    }
}
 

@Override
public void geqrf(INDArray A, INDArray R) {

    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    INDArray INFO = Nd4j.createArrayFromShapeBuffer(Nd4j.getDataBufferFactory().createInt(1),
                    Nd4j.getShapeInfoProvider().createShapeInformation(new long[] {1, 1}, A.dataType()).getFirst());

    if (R.rows() != A.columns() || R.columns() != A.columns()) {
        throw new Error("geqrf: R must be N x N (n = columns in A)");
    }
    if (A.data().dataType() == DataType.DOUBLE) {
        dgeqrf(m, n, A, R, INFO);
    } else if (A.data().dataType() == DataType.FLOAT) {
        sgeqrf(m, n, A, R, INFO);
    } else {
        throw new UnsupportedOperationException();
    }

    if (INFO.getInt(0) < 0) {
        throw new Error("Parameter #" + INFO.getInt(0) + " to getrf() was not valid");
    }
}
 

/**
 * Creates an out come vector from the specified inputs
 *
 * @param index       the index of the label
 * @param numOutcomes the number of possible outcomes
 * @return a binary label matrix used for supervised learning
 */
public static INDArray toOutcomeMatrix(int[] index, long numOutcomes) {
    INDArray ret = Nd4j.create(index.length, numOutcomes);
    for (int i = 0; i < ret.rows(); i++) {
        int[] nums = new int[(int) numOutcomes];
        nums[index[i]] = 1;
        ret.putRow(i, NDArrayUtil.toNDArray(nums));
    }

    return ret;
}
 

@Test
public void execBroadcastOpTimed2() throws Exception {
    Nd4j.create(1);
    System.out.println("A ----------------");
    INDArray array = Nd4j.zeros(2048, 1024);
    System.out.println("0 ----------------");
    INDArray arrayRow = Nd4j.ones(1024);

    System.out.println("1 ----------------");

    float sum = (float) array.sumNumber().doubleValue();
    float sum2 = (float) arrayRow.sumNumber().doubleValue();

    System.out.println("2 ----------------");

    long time1 = System.nanoTime();
    for (int x = 0; x < 1000; x++) {
        array.addiRowVector(arrayRow);
    }
    long time2 = System.nanoTime();

    System.out.println("Execution time: " + ((time2 - time1) / 1000));

    for (int x = 0; x < array.rows(); x++) {
        INDArray row = array.getRow(x);
        for (int y = 0; y < array.columns(); y++) {
            assertEquals("Failed on x.y: ["+x+"."+y+"]",1000f, row.getFloat(y), 0.01);
        }
    }
}
 

@SuppressWarnings("unchecked")
@Override
public void train (Collection<Sample> samples)
{
    logger.info("Training on {} samples", samples.size());

    if (samples.isEmpty()) {
        logger.warn("No sample to retrain neural classifier");

        return;
    }

    StopWatch watch = new StopWatch("train");
    watch.start("shuffle");

    // Shuffle the collection of samples
    final List<Sample> newSamples = new ArrayList<>(samples);
    Collections.shuffle(newSamples);

    // Build raw dataset
    watch.start("getRawDataSet");

    final DataSet dataSet = getRawDataSet(newSamples);
    final INDArray features = dataSet.getFeatures();

    // Record mean and standard deviation for every feature
    watch.start("norms");
    norms = new Norms(features.mean(0), features.std(0));
    norms.stds.addi(Nd4j.scalar(Nd4j.EPS_THRESHOLD)); // Safer, to avoid later division by 0
    logger.debug("means:{}", norms.means);
    logger.debug("stds:{}", norms.stds);
    watch.start("normalize");
    normalize(features);

    // Convert features for NeuralNetwork data format
    int rows = features.rows();
    int cols = features.columns();
    logger.info("samples: {}", rows);
    logger.info("features: {}", cols);

    INDArray labels = dataSet.getLabels();
    double[][] inputs = new double[newSamples.size()][];
    double[][] desiredOutputs = new double[newSamples.size()][];
    watch.start("build input & desiredOutputs");

    for (int ig = 0; ig < rows; ig++) {
        INDArray featureRow = features.getRow(ig);
        double[] ins = new double[cols];
        inputs[ig] = ins;

        for (int j = 0; j < cols; j++) {
            ins[j] = featureRow.getDouble(j);
        }

        INDArray labelRow = labels.getRow(ig);
        double[] des = new double[SHAPE_COUNT];
        desiredOutputs[ig] = des;

        for (int j = 0; j < SHAPE_COUNT; j++) {
            des[j] = labelRow.getDouble(j);
        }
    }

    if (constants.printWatch.isSet()) {
        watch.print();
    }

    // Train
    model.train(inputs, desiredOutputs, listener, listener.getIterationPeriod());

    // Store
    store(FILE_NAME);
}
 

public static void assertRows(INDArray n, INDArray n2) {
    if (n.rows() != n2.rows())
        throw new IllegalStateException("Mis matched rows: " + n.rows() + " != " + n2.rows());
}
 

/**
 * Create from a matrix. The rows are the indices
 * The columns are the individual element in each ndarrayindex
 *
 * @param index the matrix to getFloat indices from
 * @return the indices to getFloat
 */
public static INDArrayIndex[] create(INDArray index) {

    if (index.isMatrix()) {

        if (index.rows() > Integer.MAX_VALUE)
            throw new ND4JArraySizeException();

        NDArrayIndex[] ret = new NDArrayIndex[(int) index.rows()];
        for (int i = 0; i < index.rows(); i++) {
            INDArray row = index.getRow(i);
            val nums = new long[(int) index.getRow(i).columns()];
            for (int j = 0; j < row.columns(); j++) {
                nums[j] = (int) row.getFloat(j);
            }

            NDArrayIndex idx = new NDArrayIndex(nums);
            ret[i] = idx;

        }


        return ret;

    } else if (index.isVector()) {
        long[] indices = NDArrayUtil.toLongs(index);
        return new NDArrayIndex[] {new NDArrayIndex(indices)};
    }


    throw new IllegalArgumentException("Passed in ndarray must be a matrix or a vector");

}