org.datavec.image.data.Image Java Examples

@Override
public Image asImageMatrix(InputStream is) throws IOException {
    Mat mat = streamToMat(is);
    Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
    if (image == null || image.empty()) {
        PIX pix = pixReadMem(mat.data(), mat.cols());
        if (pix == null) {
            throw new IOException("Could not decode image from input stream");
        }
        image = convert(pix);
        pixDestroy(pix);
    }
    INDArray a = asMatrix(image);
    Image i = new Image(a, image.channels(), image.rows(), image.cols());

    image.deallocate();
    return i;
}
 

@Test(timeout = 60000)
public void runAdd(TestContext testContext) throws Exception {
    BatchNumpyInputParserVerticle verticleRef = (BatchNumpyInputParserVerticle) verticle;
    NativeImageLoader nativeImageLoader = new NativeImageLoader();
    Image image = nativeImageLoader.asImageMatrix(new ClassPathResource("data/5.png").getFile());
    File tmpFile = temporary.newFile();
    byte[] npyContents = Nd4j.toNpyByteArray(image.getImage());
    try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
        IOUtils.write(npyContents, fileOutputStream);
        fileOutputStream.flush();
    }


    given().port(port)
            .multiPart("input1", tmpFile)
            .when().post("/")
            .then().statusCode(200);
    assertNotNull("Inputs were null. This means parsing failed.", verticleRef.getBatch());
    assertTrue(verticleRef.getBatch().length >= 1);
    assertNotNull(verticleRef.getBatch());
}
 

@Test(timeout = 60000)
public void runAdd(TestContext testContext) throws Exception {
    BatchNd4jInputParserVerticle verticleRef = (BatchNd4jInputParserVerticle) verticle;
    NativeImageLoader nativeImageLoader = new NativeImageLoader();
    Image image = nativeImageLoader.asImageMatrix(new ClassPathResource("data/5.png").getFile());
    File tmpFile = temporary.newFile();
    BinarySerde.writeArrayToDisk(image.getImage(), tmpFile);

    given().port(port)
            .multiPart("input1", tmpFile)
            .when().post("/")
            .then().statusCode(200);
    assertNotNull("Inputs were null. This means parsing failed.", verticleRef.getBatch());
    assertTrue(verticleRef.getBatch().length >= 1);
    assertNotNull(verticleRef.getBatch());
}
 

@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
    Mat mat = streamToMat(inputStream);
    Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
    if (image == null || image.empty()) {
        PIX pix = pixReadMem(mat.data(), mat.cols());
        if (pix == null) {
            throw new IOException("Could not decode image from input stream");
        }
        image = convert(pix);
        pixDestroy(pix);
    }
    INDArray a = asMatrix(image);
    if(!nchw)
        a = a.permute(0,2,3,1);     //NCHW to NHWC
    Image i = new Image(a, image.channels(), image.rows(), image.cols());

    image.deallocate();
    return i;
}
 

@Override
public Image asImageMatrix(InputStream is) throws IOException {
    Mat mat = streamToMat(is);
    Mat image = imdecode(mat, CV_LOAD_IMAGE_ANYDEPTH | CV_LOAD_IMAGE_ANYCOLOR);
    if (image == null || image.empty()) {
        PIX pix = pixReadMem(mat.data(), mat.cols());
        if (pix == null) {
            throw new IOException("Could not decode image from input stream");
        }
        image = convert(pix);
        pixDestroy(pix);
    }
    INDArray a = asMatrix(image);
    Image i = new Image(a, image.channels(), image.rows(), image.cols());

    image.deallocate();
    return i;
}
 

@Test
public void runFaceDetector(TestContext testContext) throws Exception {
    File imageFile = Paths.get(new ClassPathResource(".").getFile().getAbsolutePath(), "inference/onnx/data/1.jpg").toFile();

    if (!imageFile.exists()) {
        FileUtils.copyURLToFile(new URL("https://github.com/KonduitAI/konduit-serving-examples/raw/master/data/facedetector/1.jpg"), imageFile);
    }
    NativeImageLoader nativeImageLoader = new NativeImageLoader(240, 320);
    Image image = nativeImageLoader.asImageMatrix(imageFile);

    INDArray contents = image.getImage();

    byte[] npyContents = Nd4j.toNpyByteArray(contents);

    File inputFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(inputFile, npyContents);

    Response response = given().port(port)
            .multiPart("input", inputFile)
            .body(npyContents)
            .post("nd4j/numpy")
            .andReturn();

    assertEquals("Response failed", 200, response.getStatusCode());

    JsonObject output = new JsonObject(response.asString());

    assertTrue(output.containsKey("scores"));
    assertTrue(output.containsKey("boxes"));

    INDArray scores = ObjectMappers.fromJson(output.getJsonObject("scores").encode(), NDArrayOutput.class).getNdArray();
    assertEquals(0.9539676, scores.getFloat(0), 1e-6);
    assertArrayEquals(new long[]{1, 8840}, scores.shape());

    INDArray boxes = ObjectMappers.fromJson(output.getJsonObject("boxes").encode(), NDArrayOutput.class).getNdArray();
    assertEquals(0.002913665, boxes.getFloat(0), 1e-6);
    assertArrayEquals(new long[]{1, 17680}, boxes.shape());
}
 

@Override
public Image asImageMatrix(File f, boolean nchw) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Override
public Image asImageMatrix(File f, boolean nchw) throws IOException {
    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
        return asImageMatrix(bis, nchw);
    }
}
 

@Override
public Image asImageMatrix(InputStream is) throws IOException {
    return asImageMatrix(is, true);
}
 

/** As per {@link #asMatrix(File)} but as an {@link Image}*/
public abstract Image asImageMatrix(File f) throws IOException;
 

/** As per {@link #asMatrix(File, boolean)} but as an {@link Image}*/
public abstract Image asImageMatrix(File f, boolean nchw) throws IOException;
 

/** As per {@link #asMatrix(InputStream)} but as an {@link Image}*/
public abstract Image asImageMatrix(InputStream inputStream) throws IOException;
 

/** As per {@link #asMatrix(InputStream, boolean)} but as an {@link Image}*/
public abstract Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException;
 

@Override
public Image asImageMatrix(File f) throws IOException {
    throw new UnsupportedOperationException();
}
 

private void label(Image image, List<ImageObject> objectsThisImg, INDArray outLabel, int exampleNum) {
    int oW = image.getOrigW();
    int oH = image.getOrigH();

    int W = oW;
    int H = oH;

    //put the label data into the output label array
    for (ImageObject io : objectsThisImg) {
        double cx = io.getXCenterPixels();
        double cy = io.getYCenterPixels();
        if (imageTransform != null) {
            W = imageTransform.getCurrentImage().getWidth();
            H = imageTransform.getCurrentImage().getHeight();

            float[] pts = imageTransform.query(io.getX1(), io.getY1(), io.getX2(), io.getY2());

            int minX = Math.round(Math.min(pts[0], pts[2]));
            int maxX = Math.round(Math.max(pts[0], pts[2]));
            int minY = Math.round(Math.min(pts[1], pts[3]));
            int maxY = Math.round(Math.max(pts[1], pts[3]));

            io = new ImageObject(minX, minY, maxX, maxY, io.getLabel());
            cx = io.getXCenterPixels();
            cy = io.getYCenterPixels();

            if (cx < 0 || cx >= W || cy < 0 || cy >= H) {
                continue;
            }
        }

        double[] cxyPostScaling = ImageUtils.translateCoordsScaleImage(cx, cy, W, H, width, height);
        double[] tlPost = ImageUtils.translateCoordsScaleImage(io.getX1(), io.getY1(), W, H, width, height);
        double[] brPost = ImageUtils.translateCoordsScaleImage(io.getX2(), io.getY2(), W, H, width, height);

        //Get grid position for image
        int imgGridX = (int) (cxyPostScaling[0] / width * gridW);
        int imgGridY = (int) (cxyPostScaling[1] / height * gridH);

        //Convert pixels to grid position, for TL and BR X/Y
        tlPost[0] = tlPost[0] / width * gridW;
        tlPost[1] = tlPost[1] / height * gridH;
        brPost[0] = brPost[0] / width * gridW;
        brPost[1] = brPost[1] / height * gridH;

        //Put TL, BR into label array:
        Preconditions.checkState(imgGridY >= 0 && imgGridY < outLabel.size(2), "Invalid image center in Y axis: "
                + "calculated grid location of %s, must be between 0 (inclusive) and %s (exclusive). Object label center is outside "
                + "of image bounds. Image object: %s", imgGridY, outLabel.size(2), io);
        Preconditions.checkState(imgGridX >= 0 && imgGridX < outLabel.size(3), "Invalid image center in X axis: "
                + "calculated grid location of %s, must be between 0 (inclusive) and %s (exclusive). Object label center is outside "
                + "of image bounds. Image object: %s", imgGridY, outLabel.size(2), io);

        outLabel.putScalar(exampleNum, 0, imgGridY, imgGridX, tlPost[0]);
        outLabel.putScalar(exampleNum, 1, imgGridY, imgGridX, tlPost[1]);
        outLabel.putScalar(exampleNum, 2, imgGridY, imgGridX, brPost[0]);
        outLabel.putScalar(exampleNum, 3, imgGridY, imgGridX, brPost[1]);

        //Put label class into label array: (one-hot representation)
        int labelIdx = labels.indexOf(io.getLabel());
        outLabel.putScalar(exampleNum, 4 + labelIdx, imgGridY, imgGridX, 1.0);
    }
}
 

@Override
public Image asImageMatrix(InputStream inputStream) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Test
public void testNCHW_NHWC() throws Exception {
    File f = Resources.asFile("datavec-data-image/voc/2007/JPEGImages/000005.jpg");

    ImageLoader il = new ImageLoader(32, 32, 3);

    //asMatrix(File, boolean)
    INDArray a_nchw = il.asMatrix(f);
    INDArray a_nchw2 = il.asMatrix(f, true);
    INDArray a_nhwc = il.asMatrix(f, false);

    assertEquals(a_nchw, a_nchw2);
    assertEquals(a_nchw, a_nhwc.permute(0,3,1,2));


    //asMatrix(InputStream, boolean)
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        a_nchw = il.asMatrix(is);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        a_nchw2 = il.asMatrix(is, true);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        a_nhwc = il.asMatrix(is, false);
    }
    assertEquals(a_nchw, a_nchw2);
    assertEquals(a_nchw, a_nhwc.permute(0,3,1,2));


    //asImageMatrix(File, boolean)
    Image i_nchw = il.asImageMatrix(f);
    Image i_nchw2 = il.asImageMatrix(f, true);
    Image i_nhwc = il.asImageMatrix(f, false);

    assertEquals(i_nchw.getImage(), i_nchw2.getImage());
    assertEquals(i_nchw.getImage(), i_nhwc.getImage().permute(0,3,1,2));        //NHWC to NCHW


    //asImageMatrix(InputStream, boolean)
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        i_nchw = il.asImageMatrix(is);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        i_nchw2 = il.asImageMatrix(is, true);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        i_nhwc = il.asImageMatrix(is, false);
    }
    assertEquals(i_nchw.getImage(), i_nchw2.getImage());
    assertEquals(i_nchw.getImage(), i_nhwc.getImage().permute(0,3,1,2));        //NHWC to NCHW
}
 

@Test
public void testNCHW_NHWC() throws Exception {
    File f = Resources.asFile("datavec-data-image/voc/2007/JPEGImages/000005.jpg");

    NativeImageLoader il = new NativeImageLoader(32, 32, 3);

    //asMatrix(File, boolean)
    INDArray a_nchw = il.asMatrix(f);
    INDArray a_nchw2 = il.asMatrix(f, true);
    INDArray a_nhwc = il.asMatrix(f, false);

    assertEquals(a_nchw, a_nchw2);
    assertEquals(a_nchw, a_nhwc.permute(0,3,1,2));


    //asMatrix(InputStream, boolean)
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        a_nchw = il.asMatrix(is);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        a_nchw2 = il.asMatrix(is, true);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        a_nhwc = il.asMatrix(is, false);
    }
    assertEquals(a_nchw, a_nchw2);
    assertEquals(a_nchw, a_nhwc.permute(0,3,1,2));


    //asImageMatrix(File, boolean)
    Image i_nchw = il.asImageMatrix(f);
    Image i_nchw2 = il.asImageMatrix(f, true);
    Image i_nhwc = il.asImageMatrix(f, false);

    assertEquals(i_nchw.getImage(), i_nchw2.getImage());
    assertEquals(i_nchw.getImage(), i_nhwc.getImage().permute(0,3,1,2));        //NHWC to NCHW


    //asImageMatrix(InputStream, boolean)
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        i_nchw = il.asImageMatrix(is);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        i_nchw2 = il.asImageMatrix(is, true);
    }
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        i_nhwc = il.asImageMatrix(is, false);
    }
    assertEquals(i_nchw.getImage(), i_nchw2.getImage());
    assertEquals(i_nchw.getImage(), i_nhwc.getImage().permute(0,3,1,2));        //NHWC to NCHW
}
 

@Override
public Image asImageMatrix(File f) throws IOException {
    return asImageMatrix(f, true);
}
 

public Image asImageMatrix(String filename) throws IOException {
    return asImageMatrix(new File(filename));
}
 

@Override
public Image asImageMatrix(File f) throws IOException {
    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
        return asImageMatrix(bis);
    }
}
 

@Override
public List<List<Writable>> next(int num) {
    List<File> files = new ArrayList<>(num);
    List<List<ImageObject>> objects = new ArrayList<>(num);

    for (int i = 0; i < num && hasNext(); i++) {
        File f = iter.next();
        this.currentFile = f;
        if (!f.isDirectory()) {
            files.add(f);
            objects.add(labelProvider.getImageObjectsForPath(f.getPath()));
        }
    }

    int nClasses = labels.size();

    INDArray outImg = Nd4j.create(files.size(), channels, height, width);
    INDArray outLabel = Nd4j.create(files.size(), 4 + nClasses, gridH, gridW);

    int exampleNum = 0;
    for (int i = 0; i < files.size(); i++) {
        File imageFile = files.get(i);
        this.currentFile = imageFile;
        try {
            this.invokeListeners(imageFile);
            Image image = this.imageLoader.asImageMatrix(imageFile);
            this.currentImage = image;
            Nd4j.getAffinityManager().ensureLocation(image.getImage(), AffinityManager.Location.DEVICE);

            outImg.put(new INDArrayIndex[]{point(exampleNum), all(), all(), all()}, image.getImage());

            List<ImageObject> objectsThisImg = objects.get(exampleNum);

            label(image, objectsThisImg, outLabel, exampleNum);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        exampleNum++;
    }

    if(!nchw) {
        outImg = outImg.permute(0, 2, 3, 1);        //NCHW to NHWC
        outLabel = outLabel.permute(0, 2, 3, 1);
    }
    return new NDArrayRecordBatch(Arrays.asList(outImg, outLabel));
}
 

@Override
public Image asImageMatrix(InputStream inputStream) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Override
public Image asImageMatrix(File f) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Override
public Image asImageMatrix(File f) throws IOException {
    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) {
        return asImageMatrix(bis);
    }
}
 

private void label(Image image, List<ImageObject> objectsThisImg, INDArray outLabel, int exampleNum) {
    int oW = image.getOrigW();
    int oH = image.getOrigH();

    int W = oW;
    int H = oH;

    //put the label data into the output label array
    for (ImageObject io : objectsThisImg) {
        double cx = io.getXCenterPixels();
        double cy = io.getYCenterPixels();
        if (imageTransform != null) {
            W = imageTransform.getCurrentImage().getWidth();
            H = imageTransform.getCurrentImage().getHeight();

            float[] pts = imageTransform.query(io.getX1(), io.getY1(), io.getX2(), io.getY2());

            int minX = Math.round(Math.min(pts[0], pts[2]));
            int maxX = Math.round(Math.max(pts[0], pts[2]));
            int minY = Math.round(Math.min(pts[1], pts[3]));
            int maxY = Math.round(Math.max(pts[1], pts[3]));

            io = new ImageObject(minX, minY, maxX, maxY, io.getLabel());
            cx = io.getXCenterPixels();
            cy = io.getYCenterPixels();

            if (cx < 0 || cx >= W || cy < 0 || cy >= H) {
                continue;
            }
        }

        double[] cxyPostScaling = ImageUtils.translateCoordsScaleImage(cx, cy, W, H, width, height);
        double[] tlPost = ImageUtils.translateCoordsScaleImage(io.getX1(), io.getY1(), W, H, width, height);
        double[] brPost = ImageUtils.translateCoordsScaleImage(io.getX2(), io.getY2(), W, H, width, height);

        //Get grid position for image
        int imgGridX = (int) (cxyPostScaling[0] / width * gridW);
        int imgGridY = (int) (cxyPostScaling[1] / height * gridH);

        //Convert pixels to grid position, for TL and BR X/Y
        tlPost[0] = tlPost[0] / width * gridW;
        tlPost[1] = tlPost[1] / height * gridH;
        brPost[0] = brPost[0] / width * gridW;
        brPost[1] = brPost[1] / height * gridH;

        //Put TL, BR into label array:
        outLabel.putScalar(exampleNum, 0, imgGridY, imgGridX, tlPost[0]);
        outLabel.putScalar(exampleNum, 1, imgGridY, imgGridX, tlPost[1]);
        outLabel.putScalar(exampleNum, 2, imgGridY, imgGridX, brPost[0]);
        outLabel.putScalar(exampleNum, 3, imgGridY, imgGridX, brPost[1]);

        //Put label class into label array: (one-hot representation)
        int labelIdx = labels.indexOf(io.getLabel());
        outLabel.putScalar(exampleNum, 4 + labelIdx, imgGridY, imgGridX, 1.0);
    }
}
 

@Override
public List<List<Writable>> next(int num) {
    List<File> files = new ArrayList<>(num);
    List<List<ImageObject>> objects = new ArrayList<>(num);

    for (int i = 0; i < num && hasNext(); i++) {
        File f = iter.next();
        this.currentFile = f;
        if (!f.isDirectory()) {
            files.add(f);
            objects.add(labelProvider.getImageObjectsForPath(f.getPath()));
        }
    }

    int nClasses = labels.size();

    INDArray outImg = Nd4j.create(files.size(), channels, height, width);
    INDArray outLabel = Nd4j.create(files.size(), 4 + nClasses, gridH, gridW);

    int exampleNum = 0;
    for (int i = 0; i < files.size(); i++) {
        File imageFile = files.get(i);
        this.currentFile = imageFile;
        try {
            this.invokeListeners(imageFile);
            Image image = this.imageLoader.asImageMatrix(imageFile);
            this.currentImage = image;
            Nd4j.getAffinityManager().ensureLocation(image.getImage(), AffinityManager.Location.DEVICE);

            outImg.put(new INDArrayIndex[]{point(exampleNum), all(), all(), all()}, image.getImage());

            List<ImageObject> objectsThisImg = objects.get(exampleNum);

            label(image, objectsThisImg, outLabel, exampleNum);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        exampleNum++;
    }

    return new NDArrayRecordBatch(Arrays.asList(outImg, outLabel));
}
 

@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Override
public Image asImageMatrix(File f, boolean nchw) throws IOException {
    throw new UnsupportedOperationException();
}