Java Code Examples for org.nd4j.serde.binary.BinarySerde#toArray()

The following examples show how to use org.nd4j.serde.binary.BinarySerde#toArray() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BaseLoader.java    From nd4j with Apache License 2.0 6 votes vote down vote up 
/**
 * Load an ndarray from a blob
 *
 * @param blob the blob to load from
 * @return the loaded ndarray
 */
@Override
public INDArray load(Blob blob) throws SQLException {
    if (blob == null)
        return null;
    try(InputStream is = blob.getBinaryStream()) {
        ByteBuffer direct = ByteBuffer.allocateDirect((int) blob.length());
        ReadableByteChannel readableByteChannel = Channels.newChannel(is);
        readableByteChannel.read(direct);
        Buffer byteBuffer = (Buffer) direct;
        byteBuffer.rewind();
        return BinarySerde.toArray(direct);
    } catch (Exception e) {
       throw new RuntimeException(e);
    }


}
Example 2
Source File: BaseLoader.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
/**
 * Load an ndarray from a blob
 *
 * @param blob the blob to load from
 * @return the loaded ndarray
 */
@Override
public INDArray load(Blob blob) throws SQLException {
    if (blob == null)
        return null;
    try(InputStream is = blob.getBinaryStream()) {
        ByteBuffer direct = ByteBuffer.allocateDirect((int) blob.length());
        ReadableByteChannel readableByteChannel = Channels.newChannel(is);
        readableByteChannel.read(direct);
        Buffer byteBuffer = (Buffer) direct;
        byteBuffer.rewind();
        return BinarySerde.toArray(direct);
    } catch (Exception e) {
       throw new RuntimeException(e);
    }


}
Example 3
Source File: VertxBufferNd4jInputAdapter.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
@Override
public NDArrayWritable convert(Buffer input, ConverterArgs parameters, Map<String, Object> contextData) {
    Preconditions.checkState(input.length() > 0, "Buffer appears to be empty!");
    INDArray fromNpyPointer = BinarySerde.toArray(input.getByteBuf().nioBuffer());

    //permute required
    if (parameters != null && parameters.getImageProcessingInitialLayout() != null && !parameters.getImageProcessingInitialLayout().equals(parameters.getImageProcessingRequiredLayout())) {
        fromNpyPointer = ImagePermuter.permuteOrder(fromNpyPointer, parameters.getImageProcessingInitialLayout(), parameters.getImageProcessingRequiredLayout());
    }

    return new NDArrayWritable(fromNpyPointer);
}
Example 4
Source File: VertxArrayConversion.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
/**
 * Convert a {@link Buffer}
 * to an {@link INDArray}
 * using one of three types:
 * numpy: (converts using {@link Nd4j#createNpyFromByteArray(byte[])}
 * nd4j: (converts using {@link BinarySerde#toArray(ByteBuffer)}
 * with a direct byte buffer copy (nd4j requires direct allocation
 * for byte buffers
 * json:  (converts with a straight for loop, note that this only supports matrices only)
 *
 * @param buffer the buffer to convert
 * @param type   the type of buffer
 * @return the created ndarray
 */
public static INDArray toArray(Buffer buffer, String type) {
    INDArray trueFeedback = null;
    switch (type) {
        case "numpy":
            trueFeedback = Nd4j.createNpyFromByteArray(buffer.getBytes());
            break;
        case "nd4j":
            ByteBuffer direct = ByteBuffer.allocateDirect(buffer.length());
            direct.put(buffer.getBytes());
            direct.rewind();
            trueFeedback = BinarySerde.toArray(direct);
            break;
        case "json":
            JsonArray jsonArray = new JsonArray(buffer.toString());
            INDArray arr = Nd4j.create(jsonArray.size(), jsonArray.getJsonArray(0).size());
            for (int i = 0; i < arr.rows(); i++) {
                for (int j = 0; j < arr.columns(); j++) {
                    arr.putScalar(i, j, jsonArray.getJsonArray(i).getDouble(j));
                }
            }

            trueFeedback = arr;
            break;
        default:
            throw new IllegalArgumentException("Illegal type " + type);

    }

    return trueFeedback;
}
Example 5
Source File: ArrowUtils.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
public static Writable fromEntry(int item, FieldVector from, ColumnType columnType) {
    if (from.getValueCount() < item) {
        throw new IllegalArgumentException("Index specified greater than the number of items in the vector with length " + from.getValueCount());
    } else {
        switch (columnType) {
            case Integer:
                return new IntWritable(getIntFromFieldVector(item, from));
            case Float:
                return new FloatWritable(getFloatFromFieldVector(item, from));
            case Double:
                return new DoubleWritable(getDoubleFromFieldVector(item, from));
            case Long:
                return new LongWritable(getLongFromFieldVector(item, from));
            case NDArray:
                VarBinaryVector valueVector = (VarBinaryVector) from;
                byte[] bytes = valueVector.get(item);
                ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
                direct.put(bytes);
                INDArray fromTensor = BinarySerde.toArray(direct);
                return new NDArrayWritable(fromTensor);
            case Boolean:
                BitVector bitVector = (BitVector) from;
                return new BooleanWritable(bitVector.get(item) > 0);
            case Categorical:
                VarCharVector varCharVector = (VarCharVector) from;
                return new Text(varCharVector.get(item));
            case Time:
                return new LongWritable(getLongFromFieldVector(item, from));
            case Bytes:
            default:
                throw new IllegalArgumentException("Illegal type " + from.getClass().getName());
            case String:
                VarCharVector varCharVector2 = (VarCharVector) from;
                return new Text(varCharVector2.get(item));
        }
    }
}
Example 6
Source File: SameDiffVerticleClassificationMetricsTest.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
@Test
public void runAdd(TestContext testContext) throws Exception {
    INDArray x = Nd4j.create(new float[]{1.0f, 2.0f});
    INDArray y = Nd4j.create(new float[]{2.0f, 3.0f});
    byte[] xNpy = Nd4j.toNpyByteArray(x);
    byte[] yNpy = Nd4j.toNpyByteArray(y);


    File xFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(xFile, xNpy);

    File yFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(yFile, yNpy);


    Response response = given().port(port)
            .multiPart("x", xFile)
            .multiPart("y", yFile)
            .post("/nd4j/numpy")
            .andReturn();

    assertEquals("Response failed", 200, response.getStatusCode());

    INDArray bodyResult = BinarySerde.toArray(ByteBuffer.wrap(response.getBody().asByteArray()));
    assertArrayEquals(new long[]{2}, bodyResult.shape());
    assertEquals(Nd4j.create(new float[]{3.0f, 5.0f}), bodyResult);

    given().port(port).get("/metrics")
            .then()
            .statusCode(200).and()
            .contentType("text/plain");

    Response response1 = given().port(port).get("/metrics")
            .andReturn();

    System.out.println(response1.asString());

}
Example 7
Source File: SameDiffVerticleNd4jTest.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
@Test
public void runAdd(TestContext testContext) throws Exception {
    INDArray x = Nd4j.create(new float[]{1.0f, 2.0f});
    INDArray y = Nd4j.create(new float[]{2.0f, 3.0f});
    byte[] xNpy = Nd4j.toNpyByteArray(x);
    byte[] yNpy = Nd4j.toNpyByteArray(y);


    File xFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(xFile, xNpy);

    File yFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(yFile, yNpy);


    Response response = given().port(port)
            .multiPart("x", xFile)
            .multiPart("y", yFile)
            .post("/nd4j/numpy")
            .andReturn();

    assertEquals("Response failed", 200, response.getStatusCode());

    INDArray bodyResult = BinarySerde.toArray(ByteBuffer.wrap(response.getBody().asByteArray()));
    assertArrayEquals(new long[]{2}, bodyResult.shape());
    assertEquals(Nd4j.create(new float[]{3.0f, 5.0f}), bodyResult);


}
Example 8
Source File: ArrowConverter.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
/**
 * Based on an input {@link ColumnType}
 * get an entry from a {@link FieldVector}
 *
 * @param item the row of the item to get from the column vector
 * @param from the column vector from
 * @param columnType the column type
 * @return the resulting writable
 */
public static Writable fromEntry(int item,FieldVector from,ColumnType columnType) {
    if(from.getValueCount() < item) {
        throw new IllegalArgumentException("Index specified greater than the number of items in the vector with length " + from.getValueCount());
    }

    switch(columnType) {
        case Integer:
            return new IntWritable(getIntFromFieldVector(item,from));
        case Long:
            return new LongWritable(getLongFromFieldVector(item,from));
        case Float:
            return new FloatWritable(getFloatFromFieldVector(item,from));
        case Double:
            return new DoubleWritable(getDoubleFromFieldVector(item,from));
        case Boolean:
            BitVector bitVector = (BitVector) from;
            return new BooleanWritable(bitVector.get(item) > 0);
        case Categorical:
            VarCharVector varCharVector = (VarCharVector) from;
            return new Text(varCharVector.get(item));
        case String:
            VarCharVector varCharVector2 = (VarCharVector) from;
            return new Text(varCharVector2.get(item));
        case Time:
            //TODO: need to look at closer
            return new LongWritable(getLongFromFieldVector(item,from));
        case NDArray:
            VarBinaryVector valueVector = (VarBinaryVector) from;
            byte[] bytes = valueVector.get(item);
            ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
            direct.put(bytes);
            INDArray fromTensor = BinarySerde.toArray(direct);
            return new NDArrayWritable(fromTensor);
        default:
            throw new IllegalArgumentException("Illegal type " + from.getClass().getName());
    }
}