org.nd4j.linalg.api.buffer.util.DataTypeUtil Java Examples

@Override
public boolean add(X aX) {
    if(container == null) {
        container = Nd4j.create(10);
    }
    else if(size == container.length()) {
        growCapacity(size * 2);
    }
    if(DataTypeUtil.getDtypeFromContext() == DataType.DOUBLE)
        container.putScalar(size,aX.doubleValue());
    else {
        container.putScalar(size,aX.floatValue());

    }

    size++;
    return true;
}
 

@Test
public void testRectifiedLinearDerivative() {
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    //ReLU:
    //f(x) = max(0,x)
    //Piecewise differentiable; choose f'(0) = 0
    //f'(x) = 1 if x > 0
    //f'(x) = 0 if x <= 0

    INDArray z = Nd4j.zeros(100);
    double[] expOut = new double[100];
    for (int i = 0; i < 100; i++) {
        double x = 0.1 * (i - 50);
        z.putScalar(i, x);
        expOut[i] = (x > 0 ? 1 : 0);
    }

    INDArray zPrime = Nd4j.getExecutioner()
                    .execAndReturn(new Step(z));

    for (int i = 0; i < 100; i++) {
        assertTrue(expOut[i] == zPrime.getDouble(i));
    }
}
 

@Override
public boolean add(X aX) {
    if(container == null) {
        container = Nd4j.create(10);
    }
    else if(size == container.length()) {
        growCapacity(size * 2);
    }
    if(DataTypeUtil.getDtypeFromContext() == DataBuffer.Type.DOUBLE)
        container.putScalar(size,aX.doubleValue());
    else {
        container.putScalar(size,aX.floatValue());

    }

    size++;
    return true;
}
 

@Test(timeout = 300000)
public void testTsne() throws Exception {
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    Nd4j.getRandom().setSeed(123);
    BarnesHutTsne b = new BarnesHutTsne.Builder().stopLyingIteration(10).setMaxIter(10).theta(0.5).learningRate(500)
                    .useAdaGrad(false).build();

    File f = Resources.asFile("/deeplearning4j-core/mnist2500_X.txt");
    INDArray data = Nd4j.readNumpy(f.getAbsolutePath(), "   ").get(NDArrayIndex.interval(0, 100),
            NDArrayIndex.interval(0, 784));

    ClassPathResource labels = new ClassPathResource("mnist2500_labels.txt");
    List<String> labelsList = IOUtils.readLines(labels.getInputStream()).subList(0, 100);
    b.fit(data);
    File outDir = testDir.newFolder();
    b.saveAsFile(labelsList, new File(outDir, "out.txt").getAbsolutePath());
}
 

@Test
public void randomStrangeTest() {
    DataBuffer.Type type = Nd4j.dataType();
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);

    int a=9;
    int b=2;
    int[] shapes = new int[a];
    for (int i = 0; i < a; i++) {
        shapes[i] = b;
    }
    INDArray c = Nd4j.linspace(1, (int) (100 * 1 + 1 + 2), (int) Math.pow(b, a)).reshape(shapes);
    c=c.sum(0);
    double[] d = c.data().asDouble();
    System.out.println("d: " + Arrays.toString(d));

    DataTypeUtil.setDTypeForContext(type);
}
 

public void pointerIndexerByGlobalType(Type currentType) {
    if (currentType == Type.LONG) {
        pointer = new LongPointer(length());
        setIndexer(LongRawIndexer.create((LongPointer) pointer));
        type = Type.LONG;
    } else if (currentType == Type.INT) {
        pointer = new IntPointer(length());
        setIndexer(IntIndexer.create((IntPointer) pointer));
        type = Type.INT;
    } else {
        if (DataTypeUtil.getDtypeFromContext() == Type.DOUBLE) {
            pointer = new DoublePointer(length());
            indexer = DoubleIndexer.create((DoublePointer) pointer);
        } else if (DataTypeUtil.getDtypeFromContext() == Type.FLOAT) {
            pointer = new FloatPointer(length());
            setIndexer(FloatIndexer.create((FloatPointer) pointer));
        } else if (DataTypeUtil.getDtypeFromContext() == Type.LONG) {
            pointer = new LongPointer(length());
            setIndexer(LongIndexer.create((LongPointer) pointer));
        }
    }
}
 

@Test
public void testSum(){
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    System.out.println("dType: "+ Nd4j.dataType());
    double[] d = new double[]{0.0018527202080054288, 4.860460399681257E-5, 1.0866740014537973E-4, -0.0067027589698796745, 3.137875745180366E-4, -0.004068275565124544, -0.01787441478759584, 0.008485829165871582, -2.763756128155635E-4, 2.871786662038523E-4, -0.010483973019250817, 0.007756981321203987, -0.0017533316296846166, -0.013154552997235138, 0.002664089383318023, 0.003219604745689706, -0.017140063751978196, -0.0402780728371146, -0.024062552380901596, 0.0034055167376910362, -0.0014209322438402164, -0.0019807697611663373, -2.2838830354674253E-4, 0.00802614947703168, 6.809417793628905E-5, -3.5682443741929696E-4, 2.3642116161687557E-4, 0.0020248602841291333, 0.008922488929012673, -1.8730312287067906E-6, 2.6969347916614046E-5, -1.3560804909521155E-5, -0.0019803959188298536, -0.011388435316124648, 0.009815024112605878, -6.217212819848868E-5, 7.198174495385047E-6, 7.859570666088778E-4, 3.2352438373925256E-4, 0.009310926419061586, 0.001285919484459703, -0.004614530932162568, -5.693929364499898E-4, 4.436935763832914E-5, 0.010423203318809186, 0.006593852752045009, 0.0063445124848706584, 9.737683314182195E-5, -7.002675823907349E-4, 0.0010906784650723032, -9.972152373258224E-6, 0.00871521334612937, 0.0015878927877041975, 3.5864863535235535E-4, -4.398790476749721E-5, -7.77853455185052E-5, 1.8862217750434992E-4, 0.0224868440061588, 0.0073318858545188175, -8.220926236861101E-4, -2.4336360596325374E-4, -0.0018348955616861627, 0.011423225743787646, -0.0016207645948113378, 2.289915435117371E-4, -7.122130486259979E-4, -4.94058936059287E-4, 0.004245767850547438, 2.1598406094246788E-4, 0.0014093429117757112, 3.1948093888499473E-4, -1.327894312872927E-4, 5.756401064075624E-4, -0.013501868757425933, 0.08022280647460137, -0.025763510735921924, 0.2147635435756625, -3.570893204705811E-4, 0.23699343725699218, 0.02005726530793397, 0.2233494849035487, 0.0015628679046820334, 0.03686828571588657, -0.034884254322163376, -0.04580585504492872, 0.022492109246861913, 0.6122906576027609, 0.0013512843074173794, 0.009833469844281123, -0.12754922577196826, -0.05866094108326281, 0.00786015783509335, 0.012943402024682067, -0.04138337949224019, 0.16422596234194609, -0.003224047448184361, -0.013553967826667544, 0.024567523776697443, 0.003119569763001505, 0.06676632404231841, 0.0019518418481909879, -3.546570152995131E-4, 8.843184961729061E-4, 0.02791605441470666, -0.013688049930718094, 0.03237370158354087, 7.749693316768275E-4, -0.006175397798237791, -0.001425650837729542, -7.358356122518933E-4, -5.924546696292049E-4, 2.572174974203492E-4, 0.008635399542952251, -0.00785894020636433, 0.00611004654858908, 3.849937280461072E-4, -0.0011280073492511923, -0.014039863611056342, 0.005258910284221449, -0.0012716079353840685, -0.005880609969998075, 0.03884904612026859, -0.007808162270479559, 0.13764734350512128, -0.0955607452917015, 0.01739042887598923, 0.003176716700283583, -8.845189001196553E-5, 0.059890266991132, -0.011719738031782573, -0.009720651901132008, -0.020271048497565218, 3.5861474460486776E-5, 0.003234054136867597, -0.016855942686723118, -0.04109181803561225, 0.03929335910336556, -0.002045944958743484, 4.986319734224706E-6, -2.0719501403766647E-5, 0.022377318509545937, -0.007592601387358396, 4.490315393644052E-4, 9.033852118576955E-5, 4.621091068084668E-6, -5.247702006473915E-4, -7.902654461500924E-4, -0.0011914084606579713, 0.0030085580689989877, -4.246971856810759E-4, -1.2340215512440867E-4, 0.0019671074593817285, -6.010387216740781E-4, 0.013650305790487045, -0.0011454153127967719, -0.007189180788631945, 2.870289907492301E-6, 4.3693088414999864E-4, 0.01200434591332941, -0.014509596674846678, -0.0029357117029629866, -2.1150207332328822E-4, -0.00315536512642124, 1.0374814880225154E-4, -0.0034757406691398496, 0.011599985159323294, 1.2969970680596453E-4, 9.964327556021442E-4, -0.001849649601501932, 0.002689358375591656, 0.012896200751328621, 0.007476029001401352, -0.0033194177760658377, -4.3432827454975976E-4, 3.411369387610943E-4, -4.103832908635317E-4, 0.007055642948203781, -0.0015501810107658967, -0.005752034090813254, -2.844831713420882E-4, -9.563438979460705E-5, -0.02284555356663203, -0.009025504086580169, -0.1559083024105329, 0.12294355422935457, -8.708345100849238E-4, 0.02784682111718311, 0.09887344727692746, 0.1984110780215329, -0.0019539047730033083, 0.436534119185953, -0.0022943880212978763, 0.0033303334626212217, -0.47305986663738375, 0.2870128297740214, -0.4852364244335913, -0.1966639932906117, -0.011543131716351632, -0.037961570290375855, 0.7991053621370379, -0.0965493466734368, 0.14022527291688097, -0.15353621266599798, 0.032127740955076554, -0.03391229079838272, 0.04220928870735664, -0.10022115665949234, -0.0060843857983522015, 0.05969884290137722, -0.001513774894231756, 0.003573617155056928, -0.030126515163639428, 0.006604847374388239, -0.01685524264155275, -0.015135550991685925, -0.002122525156000015};
    int[] shape = {2, 108};
    int[] stride = {108, 1};
    char order = 'c';
    INDArray arr = Nd4j.create(d, shape, stride, 0, order );

    double[] exp = new double[2];
    for( int i=0; i<shape[1]; i++ ){
        exp[0] += arr.getDouble(0,i);
        exp[1] += arr.getDouble(1,i);
    }

    System.out.println("Expected: " + Arrays.toString(exp));
    System.out.println("Actual:   " + Arrays.toString(arr.sum(1).data().asDouble()));
}
 

@Test
public void testSerializationFullArrayNd4jWriteRead() throws Exception {
    int length = 100;

    //WRITE OUT A FLOAT ARRAY
    //Hack before setting datatype - fix already in r119_various branch
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    arr.subi(50.0123456); //assures positive and negative numbers with decimal points

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(arr, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO DOUBLE and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    arr1.subi(50.0123456);

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //System.out.println(new NDArrayStrings(6).format(arr2.sub(arr1).mul(100).div(arr1)));
    assertTrue(Transforms.abs(arr1.sub(arr2).div(arr1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationOnViewsJava() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);

    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
        oos.writeObject(sub);
    }
    byte[] bytes = baos.toByteArray();
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = (INDArray) ois.readObject();
    }

    //assertEquals(sub,arr2);
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationOnViewsNd4jWriteRead() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(sub, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO DOUBLE and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //assertEquals(sub,arr2);
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
    public void testPerplexity() throws Exception {
        DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
        Nd4j.getRandom().setSeed(123);
        BarnesHutTsne b = new BarnesHutTsne.Builder().stopLyingIteration(10).setMaxIter(10).theta(0.5).learningRate(500)
                .useAdaGrad(false).build();

        DataSetIterator iter = new MnistDataSetIterator(100, true, 12345);
        INDArray data = iter.next().getFeatures();

        INDArray perplexityOutput = b.computeGaussianPerplexity(data, 30.0);
//        System.out.println(perplexityOutput);
    }
 

@Before
public void before() throws Exception {
    super.before();

    Nd4j.zeros(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);

    Nd4j.getRandom().setSeed(123);
}
 

@Test
public void testSerializationOnViewsJava() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);

    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
        oos.writeObject(sub);
    }
    byte[] bytes = baos.toByteArray();
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = (INDArray) ois.readObject();
    }

    //assertEquals(sub,arr2);
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationOnViewsNd4jWriteRead() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(sub, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO FLOAT and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //assertEquals(sub,arr2);
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Before
public void setUp() throws Exception {
    System.out.println("----------------------");
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);


    CudaEnvironment.getInstance().getConfiguration().enableDebug(true).setVerbose(true).allowMultiGPU(false);
}
 

@Before
public void setUp() {
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    CudaEnvironment.getInstance().getConfiguration()
            .allowMultiGPU(true)
            .allowCrossDeviceAccess(true)
            .enableDebug(true)
            .setMaximumGridSize(512)
            .setMaximumBlockSize(256)
            .setVerbose(true);
}
 

@Test
public void testDebugEdgeCase2(){
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    INDArray l1 = Nd4j.create(new double[]{-0.2585039112684677,-0.005179485353710878,0.4348343401770497,0.020356532375728764,-0.1970793298488186});
    INDArray l2 = Nd4j.create(2,l1.size(1));

    INDArray p1 = Nd4j.create(new double[]{1.3979850406519119,0.6169451410155852,1.128993957530918,0.21000426084450596,0.3171215178932696});
    INDArray p2 = Nd4j.create(2, p1.size(1));

    for( int i=0; i<2; i++ ){
        l2.putRow(i, l1);
        p2.putRow(i, p1);
    }

    INDArray norm2_1 = l1.norm2(1);
    INDArray temp1 = p1.mul(l1);
    INDArray out1 = temp1.diviColumnVector(norm2_1);

    INDArray norm2_2 = l2.norm2(1);
    INDArray temp2 = p2.mul(l2);
    INDArray out2 = temp2.diviColumnVector(norm2_2);

    System.out.println("norm2_1: " + Arrays.toString(norm2_1.data().asDouble()));
    System.out.println("norm2_2: " + Arrays.toString(norm2_2.data().asDouble()));

    System.out.println("temp1: " + Arrays.toString(temp1.data().asDouble()));
    System.out.println("temp2: " + Arrays.toString(temp2.data().asDouble()));

    //Outputs here should be identical:
    System.out.println(Arrays.toString(out1.data().asDouble()));
    System.out.println(Arrays.toString(out2.getRow(0).dup().data().asDouble()));
}
 

@Test
public void testNd4jDup() {
/* set the dType */
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);

/* create NDArray from a double[][] */
    int cnt = 0;
    double data[][] = new double[50][50];
    for (int x = 0; x < 50; x++) {
        for (int y = 0; y< 50; y++) {
            data[x][y] = cnt;
            cnt++;
        }
    }
    INDArray testNDArray = Nd4j.create(data);

/* print the first row */
    System.out.println("A: " + testNDArray.getRow(0));

/* set the dType again! */
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);

/* print the first row */
    System.out.println("B: " + testNDArray.getRow(0));

/* print the first row dup -- it should be different now! */
    System.out.println("C: " + testNDArray.getRow(0).dup());
}
 

@Test
public void testColumnSumDouble() {
    DataBuffer.Type initialType = Nd4j.dataType();
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    INDArray twoByThree = Nd4j.linspace(1, 600, 600).reshape(150, 4);
    INDArray columnVar = twoByThree.sum(0);
    INDArray assertion = Nd4j.create(new float[] {44850.0f, 45000.0f, 45150.0f, 45300.0f});
    assertEquals(getFailureMessage(), assertion, columnVar);
    DataTypeUtil.setDTypeForContext(initialType);
}
 

@Before
public void before() throws Exception {
    super.before();
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    Nd4j.getRandom().setSeed(SEED);

}
 

@Override
public X set(int i, X aX) {
    if(DataTypeUtil.getDtypeFromContext() == DataBuffer.Type.DOUBLE)
        container.putScalar(i,aX.doubleValue());
    else {
        container.putScalar(i,aX.floatValue());

    }


    return aX;
}
 

@Test
public void testSerializationFullArrayNd4jWriteRead() throws Exception {
    int length = 4;

    //WRITE OUT A DOUBLE ARRAY
    //Hack before setting datatype - fix already in r119_various branch
    Nd4j.create(1);
    val initialType = Nd4j.dataType();

    Nd4j.setDataType(DataType.DOUBLE);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 2, 2);
    arr.subi(50.0123456); //assures positive and negative numbers with decimal points

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(arr, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO FLOAT and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 2, 2);
    arr1.subi(50.0123456);

    log.info("A  ---------------");

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //System.out.println(new NDArrayStrings(6).format(arr2.sub(arr1).mul(100).div(arr1)));
    arr1 = arr1.castTo(DataType.DOUBLE);
    assertTrue(Transforms.abs(arr1.sub(arr2).div(arr1)).maxNumber().doubleValue() < 0.01);
}
 

@Test(timeout = 300000)
public void testTsne() throws Exception {
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    Nd4j.getRandom().setSeed(123);
    BarnesHutTsne b = new BarnesHutTsne.Builder().stopLyingIteration(10).setMaxIter(10).theta(0.5).learningRate(500)
                    .useAdaGrad(true).build();

    File f = Resources.asFile("/deeplearning4j-core/mnist2500_X.txt");
    INDArray data = Nd4j.readNumpy(f.getAbsolutePath(), "   ").get(NDArrayIndex.interval(0, 100),
                    NDArrayIndex.interval(0, 784));



    ClassPathResource labels = new ClassPathResource("mnist2500_labels.txt");
    List<String> labelsList = IOUtils.readLines(labels.getInputStream()).subList(0, 100);
    b.fit(data);
    File save = new File(System.getProperty("java.io.tmpdir"), "labels-" + UUID.randomUUID().toString());
    System.out.println("Saved to " + save.getAbsolutePath());
    save.deleteOnExit();
    b.saveAsFile(labelsList, save.getAbsolutePath());

    INDArray output = b.getData();
    System.out.println("Coordinates");

    UIServer server = UIServer.getInstance();
    Thread.sleep(10000000000L);
}
 

@Test
public void testSerializationOnViewsJava() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);

    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
        oos.writeObject(sub);
    }
    byte[] bytes = baos.toByteArray();
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length, DataType.DOUBLE).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = (INDArray) ois.readObject();
    }

    //assertEquals(sub,arr2);
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationFullArrayNd4jWriteRead() throws Exception {
    int length = 100;

    //WRITE OUT A FLOAT ARRAY
    //Hack before setting datatype - fix already in r119_various branch
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    arr.subi(50.0123456); //assures positive and negative numbers with decimal points

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(arr, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO DOUBLE and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length, DataType.FLOAT).reshape('c', 10, 10);
    arr1.subi(50.0123456);

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //System.out.println(new NDArrayStrings(6).format(arr2.sub(arr1).mul(100).div(arr1)));
    assertTrue(Transforms.abs(arr1.sub(arr2).div(arr1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationFullArrayJava() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    arr.subi(50.0123456); //assures positive and negative numbers with decimal points

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
        oos.writeObject(arr);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO DOUBLE and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    arr1.subi(50.0123456);

    INDArray arr2;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = (INDArray) ois.readObject();
    }

    assertEquals(arr.dataType(), arr2.dataType());
    arr1 = arr1.castTo(arr2.dataType());
    assertTrue(Transforms.abs(arr1.sub(arr2).div(arr1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationOnViewsNd4jWriteRead() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(sub, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO DOUBLE and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length, DataType.FLOAT).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //assertEquals(sub,arr2);\
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testSerializationOnViewsJava() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);

    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
        oos.writeObject(sub);
    }
    byte[] bytes = baos.toByteArray();
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length, DataType.FLOAT).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = (INDArray) ois.readObject();
    }

    //assertEquals(sub,arr2);
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

@Test
public void testCorrectness1() {
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    Nd4j.getRandom().setSeed(123);

    double[] aData = new double[]{
            0.2999816948164936, 0.26252049735806526, 0.2673853427498767, 0.8604464129156685, 0.4802652829902563, 0.10959096539488711, 0.7950242948008909, 0.5917848948003486,
            0.2738285999345498, 0.9519684328285567, 0.9690024759209738, 0.8585615547624705, 0.8087760944312002, 0.5337951589543348, 0.5960876109129123, 0.7187130179825856,
            0.4629777327445964, 0.08665909175584818, 0.7748005397731237, 0.48020186965468536, 0.24927351841378798, 0.32272599988270445, 0.306414968984427, 0.6980212149215657,
            0.7977183964212472, 0.7673513094629704, 0.1679681724796478, 0.3107359484804584, 0.021701726051792103, 0.13797462786662518, 0.8618953518813538, 0.841333838365635,
            0.5284957375170422, 0.9703367685039823, 0.677388096913733, 0.2624474979832243, 0.43740966353106536, 0.15685545957858893, 0.11072929134449871, 0.06007395961283357,
            0.4093918718557811,  0.9563909195720572, 0.5994144944480242, 0.8278927844215804, 0.38586830957105667, 0.6201844716257464, 0.7603829079070265, 0.07875691596842949,
            0.08651136699915507, 0.7445210640026082, 0.6547649514127559, 0.3384719042666908, 0.05816723105860,0.6248951423054205, 0.7431868493349041};
    INDArray data = Nd4j.createFromArray(aData).reshape(11,5);

    BarnesHutTsne b = new BarnesHutTsne.Builder().stopLyingIteration(250).setMaxIter(20).perplexity(3.0).theta(0.5).numDimension(5).
            invertDistanceMetric(false).similarityFunction(Distance.EUCLIDEAN.toString())
            .setMomentum(0.5).learningRate(200).staticInit(data).setSwitchMomentumIteration(250)
            .useAdaGrad(false).build();

    b.fit(data);

    double[] expectedData = new double[]{  63.8206,   80.4013,  -19.4424, -140.4326,  198.7239,
                                          106.1148,  -96.6273, -124.3634,   78.4174,  -83.6621,
                                         -121.8706,    3.0888, -172.8560,  255.1262,   20.7021,
                                         -120.7942,  -78.1829,   56.6021, -112.3294,  185.4084,
                                           88.5330,   78.0497,  -18.8673,  -11.0155, -175.1564,
                                        -297.8463,  174.2511, -103.8793,   72.5455,  -15.8498,
                                        -134.5235,   42.3300,  154.0391, -280.1010, -167.9765,
                                           306.9938, -150.9666,   83.4419,  -36.0877,   83.9992,
                                           245.1813,  -81.5018,  -14.8430,   16.1557,  166.8651,
                                           -65.9247, -138.1783,   72.5444,  176.3088,  -25.6732,
                                           -69.6843,  167.3360,   87.6238,  -18.5874, -187.3806};

    INDArray expectedArray = Nd4j.createFromArray(expectedData).reshape(11,5);
    for (int i = 0; i < expectedArray.rows(); ++i)
        assertArrayEquals(expectedArray.getRow(i).toDoubleVector(), b.getData().getRow(i).toDoubleVector(), 1e-2);
}
 

@Test
public void testColumnSumDouble() {
    DataType initialType = Nd4j.dataType();
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    INDArray twoByThree = Nd4j.linspace(1, 600, 600, DataType.DOUBLE).reshape(150, 4);
    INDArray columnVar = twoByThree.sum(0);
    INDArray assertion = Nd4j.create(new double[] {44850.0f, 45000.0f, 45150.0f, 45300.0f});
    assertEquals(getFailureMessage(), assertion, columnVar);
    DataTypeUtil.setDTypeForContext(initialType);
}