org.deeplearning4j.nn.layers.objdetect.YoloUtils Java Examples

private static void cropImageWithYOLOBoundingBox(ComputationGraph yolo,
                                                 Speed selectedSpeed, File file) throws Exception {
    if (file.isDirectory()) {
        return;
    }
    BufferedImage bufferedImage = ImageIO.read(file);
    INDArray features = LOADER.asMatrix(bufferedImage);
    opencv_core.Mat mat = LOADER.asMat(features);
    PRE_PROCESSOR.transform(features);
    INDArray results = yolo.outputSingle(features);
    Yolo2OutputLayer outputLayer = (Yolo2OutputLayer) yolo.getOutputLayer(0);
    List<DetectedObject> predictedObjects = outputLayer.getPredictedObjects(results, 0.5);
    YoloUtils.nms(predictedObjects, 0.5);
    Optional<DetectedObject> max = predictedObjects.stream()
            .max((o1, o2) -> ((Double) o1.getConfidence()).compareTo(o2.getConfidence()));
    createCroppedImage(mat, selectedSpeed, max.get(), file);
}
 

private DetectedObjectsBatch[] getPredictedObjects(INDArray[] outputs, double threshold, String[] outputNames, int originalHeight, int originalWidth) {
    // assuming "standard" output from TensorFlow using a "normal" YOLOv2 model
    //INDArray permuted = outputs[0].permute(0, 3, 1, 2);
    INDArray permuted = outputs[0];
    INDArray activated = YoloUtils.activate(boundingBoxPriors, permuted);

    List<DetectedObject> predictedObjects1 = YoloUtils.getPredictedObjects(boundingBoxPriors, activated, threshold, 0.4);


    DetectedObjectsBatch[] detectedObjects = new DetectedObjectsBatch[predictedObjects1.size()];

    int n = numLabels; // an arbitrary number of classes returned per object
    for (int i = 0; i < detectedObjects.length; i++) {
        DetectedObject detectedObject = predictedObjects1.get(i);
        long x = Math.round(originalWidth * predictedObjects1.get(i).getCenterX() / gridWidth);
        long y = Math.round(originalHeight * predictedObjects1.get(i).getCenterY() / gridHeight);
        long w = Math.round(originalWidth * predictedObjects1.get(i).getWidth() / gridWidth);
        long h = Math.round(originalHeight * predictedObjects1.get(i).getHeight() / gridHeight);

        detectedObjects[i] = new DetectedObjectsBatch();
        detectedObjects[i].setCenterX(x);
        detectedObjects[i].setCenterY(y);
        detectedObjects[i].setWidth(w);
        detectedObjects[i].setHeight(h);
        detectedObjects[i].setPredictedClasses(new String[]{labels.getLabel(detectedObject.getPredictedClass())});
        detectedObjects[i].setPredictedClassNumbers(new int[]{detectedObject.getPredictedClass()});
        detectedObjects[i].setConfidences(new float[]{detectedObject.getClassPredictions().getFloat(detectedObject.getPredictedClass())});

    }

    return detectedObjects;


}
 

public void predictBoundingBoxes(String windowName) throws Exception {
    long start = System.currentTimeMillis();
    Yolo2OutputLayer outputLayer = (Yolo2OutputLayer) modelsMap.get(windowName).getOutputLayer(0);
    Mat matFrame = stackMap.get(windowName).pop();
    INDArray indArray = prepareImage(matFrame);
    log.info("stack of frames size " + stackMap.get(windowName).size());
    if (indArray == null) {
        return;
    }

    INDArray results = modelsMap.get(windowName).outputSingle(indArray);
    if (results == null) {
        return;
    }
    List<DetectedObject> predictedObjects = outputLayer.getPredictedObjects(results, YOLO_DETECTION_THRESHOLD);
    YoloUtils.nms(predictedObjects, 0.5);
    List<MarkedObject> markedObjects = predictedObjects.stream()
            .filter(e -> groupMap.get(map.get(e.getPredictedClass())) != null)
            .map(e -> {
                try {
                    return new MarkedObject(e, trainCifar10Model.getEmbeddings(matFrame, e, selectedSpeed),
                            System.currentTimeMillis(), matFrame);
                } catch (Exception e1) {
                    e1.printStackTrace();
                }
                return null;
            }).collect(Collectors.toCollection(() -> new ConcurrentArrayList<>()));
    if (outputFrames) {
        for (MarkedObject markedObject : markedObjects) {
            ImageUtils.cropImageWithYolo(selectedSpeed, markedObject.getMatFrame(),
                    markedObject.getDetectedObject(), outputFrames);
        }
    }
    this.predictedObjects = markedObjects;
    log.info("stack of predictions size " + this.predictedObjects.size());
    log.info("Prediction time " + (System.currentTimeMillis() - start) / 1000d);
}
 

@Nullable
private MarkedObject findExistingCarMatch(MarkedObject obj) {
    MarkedObject minDistanceMarkedObject = null;
    for (MarkedObject predictedObject : previousPredictedObjects) {
        double distance = predictedObject.getL2Norm().distance2(obj.getL2Norm());
        if (strategy == Strategy.IoU_PLUS_ENCODINGS) {
            if (YoloUtils.iou(obj.getDetectedObject(),
                    predictedObject.getDetectedObject()) >= 0.5
                    && distance <= trackingThreshold) {
                minDistanceMarkedObject = predictedObject;
                obj.setId(minDistanceMarkedObject.getId());
                break;
            }
        } else if (strategy == Strategy.ONLY_ENCODINGS) {
            if (distance <= trackingThreshold) {
                minDistanceMarkedObject = predictedObject;
                obj.setId(minDistanceMarkedObject.getId());
                break;
            }
        } else if (strategy == Strategy.ONLY_IoU) {
            if (YoloUtils.iou(obj.getDetectedObject(),
                    predictedObject.getDetectedObject()) >= 0.4) {
                minDistanceMarkedObject = predictedObject;
                obj.setId(minDistanceMarkedObject.getId());
                break;
            }
        }
    }
    return minDistanceMarkedObject;
}
 

public void drawBoundingBoxesRectangles(Frame frame, Mat matFrame) {
    if (invalidData(frame, matFrame)) return;

    ArrayList<DetectedObject> detectedObjects = new ArrayList<>(predictedObjects);
    YoloUtils.nms(detectedObjects, 0.5);
    for (DetectedObject detectedObject : detectedObjects) {
        createBoundingBoxRectangle(matFrame, frame.imageWidth, frame.imageHeight, detectedObject);
    }

}
 

private void renderJavaFXStyle(GraphicsContext ctx) throws Exception {

        INDArray boundingBoxPriors = Nd4j.create(YOLO2.DEFAULT_PRIOR_BOXES);

        ctx.setLineWidth(3);
        ctx.setTextAlign(TextAlignment.LEFT);

        long start = System.nanoTime();
        for (int i = 0; i < 1; i++) {
            INDArray permuted = networkGlobalOutput.permute(0, 3, 1, 2);
            INDArray activated = YoloUtils.activate(boundingBoxPriors, permuted);
            List<DetectedObject> predictedObjects = YoloUtils.getPredictedObjects(boundingBoxPriors, activated, 0.6, 0.4);
            
            //System.out.println( "width: " + imageWidth );
            //System.out.println( "height: " + imageHeight );

            for (DetectedObject o : predictedObjects) {
                ClassPrediction classPrediction = labels.decodePredictions(o.getClassPredictions(), 1).get(0).get(0);
                String label = classPrediction.getLabel();
                long x = Math.round(imageWidth  * o.getCenterX() / gridWidth);
                long y = Math.round(imageHeight * o.getCenterY() / gridHeight);
                long w = Math.round(imageWidth  * o.getWidth()   / gridWidth);
                long h = Math.round(imageHeight * o.getHeight()  / gridHeight);

                System.out.println("\"" + label + "\" at [" + x + "," + y + ";" + w + "," + h + "], score = "
                                + o.getConfidence() * classPrediction.getProbability());

                double[] xy1 = o.getTopLeftXY();
                double[] xy2 = o.getBottomRightXY();
                int x1 = (int) Math.round(imageWidth  * xy1[0] / gridWidth);
                int y1 = (int) Math.round(imageHeight * xy1[1] / gridHeight);
                int x2 = (int) Math.round(imageWidth  * xy2[0] / gridWidth);
                int y2 = (int) Math.round(imageHeight * xy2[1] / gridHeight);

                int rectW = x2 - x1;
                int rectH = y2 - y1;
                //System.out.printf("%s - %d, %d, %d, %d \n", label, x1, x2, y1, y2);
                //System.out.println( "color: " + colors.get(label) );
                ctx.setStroke(colors.get(label));
                ctx.strokeRect(x1, y1, rectW, rectH);
                
                int labelWidth = label.length() * 10;
                int labelHeight = 14;
                
                ctx.setFill( colors.get(label) );
                ctx.strokeRect(x1, y1-labelHeight, labelWidth, labelHeight);
                ctx.fillRect(x1, y1 - labelHeight, labelWidth, labelHeight);
                ctx.setFill( Color.WHITE );
                ctx.fillText(label, x1 + 3, y1 - 3 );

            }

        }

        globalEndTime = System.nanoTime();
        long end = System.nanoTime();
        System.out.println("Rendering code took: " + (end - start) / 1000000 + " ms");

        System.out.println("Overall Program Time: " + (globalEndTime - globalStartTime) / 1000000 + " ms");

    }