Java Code Examples for org.nd4j.linalg.factory.Nd4j#read()

public void setup(SparkConf conf) {
    useAdaGrad = conf.getBoolean(Word2VecVariables.ADAGRAD, false);
    negative = conf.getDouble(Word2VecVariables.NEGATIVE, 5);
    numWords = conf.getInt(Word2VecVariables.NUM_WORDS, 1);
    window = conf.getInt(Word2VecVariables.WINDOW, 5);
    alpha = conf.getDouble(Word2VecVariables.ALPHA, 0.025f);
    minAlpha = conf.getDouble(Word2VecVariables.MIN_ALPHA, 1e-2f);
    totalWords = conf.getInt(Word2VecVariables.NUM_WORDS, 1);
    vectorLength = conf.getInt(Word2VecVariables.VECTOR_LENGTH, 100);
    initExpTable();

    if (negative > 0 && conf.contains(Word2VecVariables.TABLE)) {
        ByteArrayInputStream bis = new ByteArrayInputStream(conf.get(Word2VecVariables.TABLE).getBytes());
        DataInputStream dis = new DataInputStream(bis);
        table = Nd4j.read(dis);
    }

}
 

@Test
public void testZip() throws Exception {

    File testFile = File.createTempFile("adasda","Dsdasdea");

    INDArray arr = Nd4j.create(new double[]{1,2,3,4,5,6,7,8,9,0});

    final FileOutputStream fileOut = new FileOutputStream(testFile);
    final ZipOutputStream zipOut = new ZipOutputStream(fileOut);
    zipOut.putNextEntry(new ZipEntry("params"));
    Nd4j.write(zipOut, arr);
    zipOut.flush();
    zipOut.close();


    final FileInputStream fileIn = new FileInputStream(testFile);
    final ZipInputStream zipIn = new ZipInputStream(fileIn);
    ZipEntry entry = zipIn.getNextEntry();
    INDArray read = Nd4j.read(zipIn);
    zipIn.close();


    assertEquals(arr, read);
}
 

@Test
public void testThresholdSerialization1() throws Exception {
    INDArray initial = Nd4j.create(new double[] {-1.0, -2.0, 0.0, 0.0, 1.0, 1.0});
    INDArray exp_0 = Nd4j.create(new double[] {-1.0 + 1e-3, -2.0 + 1e-3, 0.0, 0.0, 1.0 - 1e-3, 1.0 - 1e-3});
    INDArray exp_1 = Nd4j.create(new double[] {-1e-3, -1e-3, 0.0, 0.0, 1e-3, 1e-3});

    //Nd4j.getCompressor().getCompressor("THRESHOLD").configure(1e-3);
    INDArray compressed = Nd4j.getExecutioner().thresholdEncode(initial, 1e-3f);

    assertEquals(exp_0, initial);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    Nd4j.write(baos, compressed);

    INDArray serialized = Nd4j.read(new ByteArrayInputStream(baos.toByteArray()));

    INDArray decompressed_copy = Nd4j.create(initial.length());
    Nd4j.getExecutioner().thresholdDecode(serialized, decompressed_copy);

    assertEquals(exp_1, decompressed_copy);
}
 

@Test
@Ignore("AB 2019/05/23 - Failing on linux-x86_64-cuda-9.2 - see issue #7657")
public void testYoloHouseNumber() throws Exception {

    File f = Resources.asFile("regression_testing/100b3/HouseNumberDetection_100b3.bin");
    ComputationGraph net = ComputationGraph.load(f, true);

    int nBoxes = 5;
    int nClasses = 10;

    ConvolutionLayer cl = (ConvolutionLayer)((LayerVertex)net.getConfiguration().getVertices().get("convolution2d_9")).getLayerConf().getLayer();
    assertEquals(nBoxes * (5 + nClasses), cl.getNOut());
    assertEquals(new ActivationIdentity(), cl.getActivationFn());
    assertEquals(ConvolutionMode.Same, cl.getConvolutionMode());
    assertEquals(new WeightInitXavier(), cl.getWeightInitFn());
    assertArrayEquals(new int[]{1,1}, cl.getKernelSize());
    assertArrayEquals(new int[]{1,1}, cl.getKernelSize());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100b3/HouseNumberDetection_Output_100b3.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f2))){
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100b3/HouseNumberDetection_Input_100b3.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f3))){
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.outputSingle(in);

    boolean eq = outExp.equalsWithEps(outAct.castTo(outExp.dataType()), 1e-3);
    assertTrue(eq);
}
 

@Test
@Ignore // this should be commented out, since it requires approx 10GB ram to run
public void testLargeArraySerDe_2() throws Exception {
    INDArray arrayA = Nd4j.createUninitialized(100000, 12500);
    log.info("Shape: {}; Length: {}", arrayA.shape(), arrayA.length());

    val tmpFile = File.createTempFile("sdsds", "sdsd");
    tmpFile.deleteOnExit();

    log.info("Starting serialization...");
    val sS = System.currentTimeMillis();
    try (val fos = new FileOutputStream(tmpFile); val bos = new BufferedOutputStream(fos); val dos = new DataOutputStream(bos)) {
        Nd4j.write(arrayA, dos);
        arrayA = null;
        System.gc();
    }
    System.gc();

    val sE = System.currentTimeMillis();

    log.info("Starting deserialization...");
    val dS = System.currentTimeMillis();
    try (val fis = new FileInputStream(tmpFile); val bis = new BufferedInputStream(fis); val dis = new DataInputStream(bis)) {
        arrayA = Nd4j.read(dis);
    }
    val dE = System.currentTimeMillis();

    log.info("Timings: {Ser : {} ms; De: {} ms;}", sE - sS, dE - dS);
}
 

@Test
public void testVae() throws Exception {

    File f = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_100a.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    VariationalAutoencoder l0 = (VariationalAutoencoder) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationLReLU(), l0.getActivationFn());
    assertEquals(32, l0.getNOut());
    assertArrayEquals(new int[]{256, 256}, l0.getEncoderLayerSizes());
    assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes());
            assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new WeightDecay(1e-4, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new Adam(0.05), l0.getIUpdater());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_Output_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f2))){
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_Input_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f3))){
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

private DataSet read(int idx) throws IOException {
    BufferedInputStream bis = new BufferedInputStream(new FileInputStream(paths.get(idx)[0]));
    DataInputStream dis = new DataInputStream(bis);
    BufferedInputStream labelInputStream = new BufferedInputStream(new FileInputStream(paths.get(idx)[1]));
    DataInputStream labelDis = new DataInputStream(labelInputStream);
    DataSet d = new DataSet(Nd4j.read(dis), Nd4j.read(labelDis));
    dis.close();
    labelDis.close();
    return d;
}
 

/**
 * Create an ndarray from a base 64
 * representation
 * @param base64 the base 64 to convert
 * @return the ndarray from base 64
 */
public static INDArray fromBase64(String base64) {
    byte[] arr = Base64.decodeBase64(base64);
    ByteArrayInputStream bis = new ByteArrayInputStream(arr);
    DataInputStream dis = new DataInputStream(bis);
    return Nd4j.read(dis);
}
 

@Test
public void testSerializationOnViewsNd4jWriteRead() throws Exception {
    int length = 100;
    INDArray arrC = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray arrF = Nd4j.linspace(1, length, length).reshape('f', 10, 10);

    INDArray subC = arrC.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));
    INDArray subF = arrF.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(subC, dos);
    }
    byte[] bytesC = baos.toByteArray();

    baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(subF, dos);
    }
    byte[] bytesF = baos.toByteArray();


    INDArray arr2C;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytesC))) {
        arr2C = Nd4j.read(dis);
    }

    INDArray arr2F;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytesF))) {
        arr2F = Nd4j.read(dis);
    }

    assertEquals(subC, arr2C);
    assertEquals(subF, arr2F);
}
 

@Test
public void testYoloHouseNumber() throws Exception {

    File f = Resources.asFile("regression_testing/100b6/HouseNumberDetection_100b6.bin");
    ComputationGraph net = ComputationGraph.load(f, true);

    int nBoxes = 5;
    int nClasses = 10;

    ConvolutionLayer cl = (ConvolutionLayer) ((LayerVertex) net.getConfiguration().getVertices()
            .get("convolution2d_9")).getLayerConf().getLayer();
    assertEquals(nBoxes * (5 + nClasses), cl.getNOut());
    assertEquals(new ActivationIdentity(), cl.getActivationFn());
    assertEquals(ConvolutionMode.Same, cl.getConvolutionMode());
    assertEquals(new WeightInitXavier(), cl.getWeightInitFn());
    assertArrayEquals(new int[]{1, 1}, cl.getKernelSize());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100b6/HouseNumberDetection_Output_100b6.bin");
    try (DataInputStream dis = new DataInputStream(new FileInputStream(f2))) {
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100b6/HouseNumberDetection_Input_100b6.bin");
    try (DataInputStream dis = new DataInputStream(new FileInputStream(f3))) {
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.outputSingle(in);

    boolean eq = outExp.equalsWithEps(outAct.castTo(outExp.dataType()), 1e-3);
    assertTrue(eq);
}
 

@Test
public void testReadWrite() throws Exception {
    INDArray write = Nd4j.linspace(1, 4, 4);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    Nd4j.write(write, dos);

    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream dis = new DataInputStream(bis);
    INDArray read = Nd4j.read(dis);
    assertEquals(write, read);

}
 

private static NormalizerStats readMinMaxStats(DataInputStream dis) throws IOException {
    return new MinMaxStats(Nd4j.read(dis), Nd4j.read(dis));
}
 

@Test
public void testAutoDecompression2() throws Exception {
    INDArray array = Nd4j.linspace(1, 10, 11, DataType.DOUBLE);

    INDArray compressed = Nd4j.getCompressor().compress(array, "GZIP");

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    Nd4j.write(bos, compressed);

    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());

    System.out.println("Restoring -------------------------");

    INDArray result = Nd4j.read(bis);

    System.out.println("Decomp -------------------------");

    INDArray decomp = Nd4j.getCompressor().decompress(result);

    assertEquals(array, decomp);
}
 

@Test
public void testIntermediate1() throws Exception {
    Nd4j.create(1);

    val tg = TFGraphMapper.getInstance().importGraph(new ClassPathResource("tf_graphs/tensorflow_inception_graph.pb").getInputStream());

    assertTrue(tg.getVariable("input") != null);
    // assertTrue(tg.getVariableSpace().getVariable("input").isPlaceholder());

    val ipod = Nd4j.read(new DataInputStream(new ClassPathResource("tf_graphs/ipod.nd4").getInputStream()));

    tg.updateVariable("input",ipod);

    val buffer = tg.asFlatBuffers();
    assertNotNull(buffer);

}
 

@Test
public void testLSTM() throws Exception {

    File f = Resources.asFile("regression_testing/100b4/GravesLSTMCharModelingExample_100b4.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    LSTM l0 = (LSTM) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationTanH(), l0.getActivationFn());
    assertEquals(200, l0.getNOut());
    assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0));
    assertEquals(new Adam(0.005), l0.getIUpdater());

    LSTM l1 = (LSTM) net.getLayer(1).conf().getLayer();
    assertEquals(new ActivationTanH(), l1.getActivationFn());
    assertEquals(200, l1.getNOut());
    assertEquals(new WeightInitXavier(), l1.getWeightInitFn());
    assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l1));
    assertEquals(new Adam(0.005), l1.getIUpdater());

    RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).conf().getLayer();
    assertEquals(new ActivationSoftmax(), l2.getActivationFn());
    assertEquals(77, l2.getNOut());
    assertEquals(new WeightInitXavier(), l2.getWeightInitFn());
    assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l2));
    assertEquals(new Adam(0.005), l2.getIUpdater());

    assertEquals(BackpropType.TruncatedBPTT, net.getLayerWiseConfigurations().getBackpropType());
    assertEquals(50, net.getLayerWiseConfigurations().getTbpttBackLength());
    assertEquals(50, net.getLayerWiseConfigurations().getTbpttFwdLength());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100b4/GravesLSTMCharModelingExample_Output_100b4.bin");
    try (DataInputStream dis = new DataInputStream(new FileInputStream(f2))) {
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100b4/GravesLSTMCharModelingExample_Input_100b4.bin");
    try (DataInputStream dis = new DataInputStream(new FileInputStream(f3))) {
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

private static NormalizerStats readDistributionStats(DataInputStream dis) throws IOException {
    return new DistributionStats(Nd4j.read(dis), Nd4j.read(dis));
}
 

private static NormalizerStats readMinMaxStats(DataInputStream dis) throws IOException {
    return new MinMaxStats(Nd4j.read(dis), Nd4j.read(dis));
}
 

@Test
public void testGravesLSTM() throws Exception {

    File f = Resources.asFile("regression_testing/100a/GravesLSTMCharModelingExample_100a.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    GravesLSTM l0 = (GravesLSTM) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationTanH(), l0.getActivationFn());
    assertEquals(200, l0.getNOut());
    assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new RmsProp(0.1), l0.getIUpdater());

    GravesLSTM l1 = (GravesLSTM) net.getLayer(1).conf().getLayer();
    assertEquals(new ActivationTanH(), l1.getActivationFn());
    assertEquals(200, l1.getNOut());
    assertEquals(new WeightInitXavier(), l1.getWeightInitFn());
    assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l1));
    assertEquals(new RmsProp(0.1), l1.getIUpdater());

    RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).conf().getLayer();
    assertEquals(new ActivationSoftmax(), l2.getActivationFn());
    assertEquals(77, l2.getNOut());
    assertEquals(new WeightInitXavier(), l2.getWeightInitFn());
    assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new RmsProp(0.1), l0.getIUpdater());

    assertEquals(BackpropType.TruncatedBPTT, net.getLayerWiseConfigurations().getBackpropType());
    assertEquals(50, net.getLayerWiseConfigurations().getTbpttBackLength());
    assertEquals(50, net.getLayerWiseConfigurations().getTbpttFwdLength());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100a/GravesLSTMCharModelingExample_Output_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f2))){
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100a/GravesLSTMCharModelingExample_Input_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f3))){
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

@Test
public void testHalfToFloat1() throws Exception {
    File tempFile = File.createTempFile("dsadasd","dsdfasd");
    tempFile.deleteOnExit();

    INDArray array = Nd4j.linspace(1, 100, 100);

    DataOutputStream stream = new DataOutputStream(new FileOutputStream(tempFile));

    Nd4j.write(array, stream);

    DataInputStream dis = new DataInputStream(new FileInputStream(tempFile));

    INDArray restoredFP16 = Nd4j.read(dis);

    //assertEquals(array, restoredFP16);


    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    assertEquals(DataBuffer.Type.FLOAT, Nd4j.dataType());
    log.error("--------------------");

    dis = new DataInputStream(new FileInputStream(tempFile));
    INDArray expFP32 = Nd4j.linspace(1, 100, 100);
    INDArray restoredFP32 = Nd4j.read(dis);

    CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext();

    assertTrue(AtomicAllocator.getInstance().getPointer(expFP32, context) instanceof FloatPointer);
    assertTrue(AtomicAllocator.getInstance().getPointer(restoredFP32, context) instanceof FloatPointer);

    assertEquals(DataBuffer.Type.FLOAT, expFP32.data().dataType());
    assertEquals(DataBuffer.Type.FLOAT, restoredFP32.data().dataType());

    assertEquals(expFP32, restoredFP32);

    DataTypeUtil.setDTypeForContext(DataBuffer.Type.HALF);
}
 

@Test
public void testAutoDecompression1() throws Exception {
    INDArray array = Nd4j.linspace(1, 250, 250);

    INDArray compressed = Nd4j.getCompressor().compress(array, "UINT8");

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    Nd4j.write(bos, compressed);

    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());

    INDArray result = Nd4j.read(bis);

    assertEquals(array, result);
}