Java Code Examples for org.nd4j.evaluation.classification.ROC#eval()

@Test
public void testRocBasicSingleClass() {
    //1 output here - single probability value (sigmoid)

    //add 0.001 to avoid numerical/rounding issues (float vs. double, etc)
    INDArray predictions =
                    Nd4j.create(new double[] {0.001, 0.101, 0.201, 0.301, 0.401, 0.501, 0.601, 0.701, 0.801, 0.901},
                                    new int[] {10, 1});

    INDArray actual = Nd4j.create(new double[] {0, 0, 0, 0, 0, 1, 1, 1, 1, 1}, new int[] {10, 1});

    ROC roc = new ROC(10);
    roc.eval(actual, predictions);

    RocCurve rocCurve = roc.getRocCurve();

    assertEquals(11, rocCurve.getThreshold().length); //0 + 10 steps
    for (int i = 0; i < 11; i++) {
        double expThreshold = i / 10.0;
        assertEquals(expThreshold, rocCurve.getThreshold(i), 1e-5);

        //            System.out.println("t=" + expThreshold + "\t" + v.getFalsePositiveRate() + "\t" + v.getTruePositiveRate());

        double efpr = expFPR.get(expThreshold);
        double afpr = rocCurve.getFalsePositiveRate(i);
        assertEquals(efpr, afpr, 1e-5);

        double etpr = expTPR.get(expThreshold);
        double atpr = rocCurve.getTruePositiveRate(i);
        assertEquals(etpr, atpr, 1e-5);
    }

    //Expect AUC == 1.0 here
    double auc = roc.calculateAUC();
    assertEquals(1.0, auc, 1e-6);
}
 

@Test
public void testCompareRocAndRocMultiClass() {
    Nd4j.getRandom().setSeed(12345);

    //For 2 class case: ROC and Multi-class ROC should be the same...
    int nExamples = 200;
    INDArray predictions = Nd4j.rand(nExamples, 2);
    INDArray tempSum = predictions.sum(1);
    predictions.diviColumnVector(tempSum);

    INDArray labels = Nd4j.create(nExamples, 2);
    Random r = new Random(12345);
    for (int i = 0; i < nExamples; i++) {
        labels.putScalar(i, r.nextInt(2), 1.0);
    }

    for (int numSteps : new int[] {30, 0}) { //Steps = 0: exact
        ROC roc = new ROC(numSteps);
        roc.eval(labels, predictions);

        ROCMultiClass rocMultiClass = new ROCMultiClass(numSteps);
        rocMultiClass.eval(labels, predictions);

        double auc = roc.calculateAUC();
        double auc1 = rocMultiClass.calculateAUC(1);

        assertEquals(auc, auc1, 1e-6);
    }
}
 

@Test
public void testPrecisionRecallCurveConfusion() {
    //Sanity check: values calculated from the confusion matrix should match the PR curve values

    for (boolean removeRedundantPts : new boolean[] {true, false}) {
        ROC r = new ROC(0, removeRedundantPts);

        INDArray labels = Nd4j.getExecutioner()
                        .exec(new BernoulliDistribution(Nd4j.createUninitialized(DataType.DOUBLE,100, 1), 0.5));
        INDArray probs = Nd4j.rand(100, 1);

        r.eval(labels, probs);

        PrecisionRecallCurve prc = r.getPrecisionRecallCurve();
        int nPoints = prc.numPoints();

        for (int i = 0; i < nPoints; i++) {
            PrecisionRecallCurve.Confusion c = prc.getConfusionMatrixAtPoint(i);
            PrecisionRecallCurve.Point p = c.getPoint();

            int tp = c.getTpCount();
            int fp = c.getFpCount();
            int fn = c.getFnCount();

            double prec = tp / (double) (tp + fp);
            double rec = tp / (double) (tp + fn);

            //Handle edge cases:
            if (tp == 0 && fp == 0) {
                prec = 1.0;
            }

            assertEquals(p.getPrecision(), prec, 1e-6);
            assertEquals(p.getRecall(), rec, 1e-6);
        }
    }
}
 

@Test
    public void testROCBinary() {
        //Compare ROCBinary to ROC class

        DataType dtypeBefore = Nd4j.defaultFloatingPointType();
        ROCBinary first30 = null;
        ROCBinary first0 = null;
        String sFirst30 = null;
        String sFirst0 = null;
        try {
            for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.INT}) {
//            for (DataType globalDtype : new DataType[]{DataType.HALF}) {
                Nd4j.setDefaultDataTypes(globalDtype, globalDtype.isFPType() ? globalDtype : DataType.DOUBLE);
                for (DataType lpDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
                    String msg = "globalDtype=" + globalDtype + ", labelPredictionsDtype=" + lpDtype;

                    int nExamples = 50;
                    int nOut = 4;
                    long[] shape = {nExamples, nOut};

                    for (int thresholdSteps : new int[]{30, 0}) { //0 == exact

                        Nd4j.getRandom().setSeed(12345);
                        INDArray labels =
                                Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(DataType.DOUBLE, shape), 0.5)).castTo(lpDtype);

                        Nd4j.getRandom().setSeed(12345);
                        INDArray predicted = Nd4j.rand(DataType.DOUBLE, shape).castTo(lpDtype);

                        ROCBinary rb = new ROCBinary(thresholdSteps);

                        for (int xe = 0; xe < 2; xe++) {
                            rb.eval(labels, predicted);

                            //System.out.println(rb.stats());

                            double eps = lpDtype == DataType.HALF ? 1e-2 : 1e-6;
                            for (int i = 0; i < nOut; i++) {
                                INDArray lCol = labels.getColumn(i, true);
                                INDArray pCol = predicted.getColumn(i, true);


                                ROC r = new ROC(thresholdSteps);
                                r.eval(lCol, pCol);

                                double aucExp = r.calculateAUC();
                                double auc = rb.calculateAUC(i);

                                assertEquals(msg, aucExp, auc, eps);

                                long apExp = r.getCountActualPositive();
                                long ap = rb.getCountActualPositive(i);
                                assertEquals(msg, ap, apExp);

                                long anExp = r.getCountActualNegative();
                                long an = rb.getCountActualNegative(i);
                                assertEquals(anExp, an);

                                PrecisionRecallCurve pExp = r.getPrecisionRecallCurve();
                                PrecisionRecallCurve p = rb.getPrecisionRecallCurve(i);

                                assertEquals(msg, pExp, p);
                            }

                            String s = rb.stats();

                            if(thresholdSteps == 0){
                                if(first0 == null) {
                                    first0 = rb;
                                    sFirst0 = s;
                                } else if(lpDtype != DataType.HALF) {   //Precision issues with FP16
                                    assertEquals(msg, sFirst0, s);
                                    assertEquals(first0, rb);
                                }
                            } else {
                                if(first30 == null) {
                                    first30 = rb;
                                    sFirst30 = s;
                                } else if(lpDtype != DataType.HALF) {   //Precision issues with FP16
                                    assertEquals(msg, sFirst30, s);
                                    assertEquals(first30, rb);
                                }
                            }

//                            rb.reset();
                            rb = new ROCBinary(thresholdSteps);
                        }
                    }
                }
            }
        } finally {
            Nd4j.setDefaultDataTypes(dtypeBefore, dtypeBefore);
        }
    }
 

@Test
public void testNewInstances() {
    boolean print = true;
    Nd4j.getRandom().setSeed(12345);

    Evaluation evaluation = new Evaluation();
    EvaluationBinary evaluationBinary = new EvaluationBinary();
    ROC roc = new ROC(2);
    ROCBinary roc2 = new ROCBinary(2);
    ROCMultiClass roc3 = new ROCMultiClass(2);
    RegressionEvaluation regressionEvaluation = new RegressionEvaluation();
    EvaluationCalibration ec = new EvaluationCalibration();


    IEvaluation[] arr = new IEvaluation[] {evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec};

    INDArray evalLabel1 = Nd4j.create(10, 3);
    for (int i = 0; i < 10; i++) {
        evalLabel1.putScalar(i, i % 3, 1.0);
    }
    INDArray evalProb1 = Nd4j.rand(10, 3);
    evalProb1.diviColumnVector(evalProb1.sum(1));

    evaluation.eval(evalLabel1, evalProb1);
    roc3.eval(evalLabel1, evalProb1);
    ec.eval(evalLabel1, evalProb1);

    INDArray evalLabel2 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5));
    INDArray evalProb2 = Nd4j.rand(10, 3);
    evaluationBinary.eval(evalLabel2, evalProb2);
    roc2.eval(evalLabel2, evalProb2);

    INDArray evalLabel3 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5));
    INDArray evalProb3 = Nd4j.rand(10, 1);
    roc.eval(evalLabel3, evalProb3);

    INDArray reg1 = Nd4j.rand(10, 3);
    INDArray reg2 = Nd4j.rand(10, 3);

    regressionEvaluation.eval(reg1, reg2);

    Evaluation evaluation2 = evaluation.newInstance();
    EvaluationBinary evaluationBinary2 = evaluationBinary.newInstance();
    ROC roc_2 = roc.newInstance();
    ROCBinary roc22 = roc2.newInstance();
    ROCMultiClass roc32 = roc3.newInstance();
    RegressionEvaluation regressionEvaluation2 = regressionEvaluation.newInstance();
    EvaluationCalibration ec2 = ec.newInstance();

    IEvaluation[] arr2 = new IEvaluation[] {evaluation2, evaluationBinary2, roc_2, roc22, roc32, regressionEvaluation2, ec2};

    evaluation2.eval(evalLabel1, evalProb1);
    roc32.eval(evalLabel1, evalProb1);
    ec2.eval(evalLabel1, evalProb1);

    evaluationBinary2.eval(evalLabel2, evalProb2);
    roc22.eval(evalLabel2, evalProb2);

    roc_2.eval(evalLabel3, evalProb3);

    regressionEvaluation2.eval(reg1, reg2);

    for (int i = 0 ; i < arr.length ; i++) {

        IEvaluation e = arr[i];
        IEvaluation e2 = arr2[i];
        assertEquals("Json not equal ", e.toJson(), e2.toJson());
        assertEquals("Stats not equal ", e.stats(), e2.stats());
    }
}
 

@Test
public void testRocBasic() {
    //2 outputs here - probability distribution over classes (softmax)
    INDArray predictions = Nd4j.create(new double[][] {{1.0, 0.001}, //add 0.001 to avoid numerical/rounding issues (float vs. double, etc)
                    {0.899, 0.101}, {0.799, 0.201}, {0.699, 0.301}, {0.599, 0.401}, {0.499, 0.501}, {0.399, 0.601},
                    {0.299, 0.701}, {0.199, 0.801}, {0.099, 0.901}});

    INDArray actual = Nd4j.create(new double[][] {{1, 0}, {1, 0}, {1, 0}, {1, 0}, {1, 0}, {0, 1}, {0, 1}, {0, 1},
                    {0, 1}, {0, 1}});

    ROC roc = new ROC(10);
    roc.eval(actual, predictions);

    RocCurve rocCurve = roc.getRocCurve();

    assertEquals(11, rocCurve.getThreshold().length); //0 + 10 steps
    for (int i = 0; i < 11; i++) {
        double expThreshold = i / 10.0;
        assertEquals(expThreshold, rocCurve.getThreshold(i), 1e-5);

        //            System.out.println("t=" + expThreshold + "\t" + v.getFalsePositiveRate() + "\t" + v.getTruePositiveRate());

        double efpr = expFPR.get(expThreshold);
        double afpr = rocCurve.getFalsePositiveRate(i);
        assertEquals(efpr, afpr, 1e-5);

        double etpr = expTPR.get(expThreshold);
        double atpr = rocCurve.getTruePositiveRate(i);
        assertEquals(etpr, atpr, 1e-5);
    }


    //Expect AUC == 1.0 here
    double auc = roc.calculateAUC();
    assertEquals(1.0, auc, 1e-6);

    // testing reset now
    roc.reset();
    roc.eval(actual, predictions);
    auc = roc.calculateAUC();
    assertEquals(1.0, auc, 1e-6);
}
 

@Test
public void testRoc() {
    //Previous tests allowed for a perfect classifier with right threshold...

    INDArray labels = Nd4j.create(new double[][] {{0, 1}, {0, 1}, {1, 0}, {1, 0}, {1, 0}});

    INDArray prediction = Nd4j.create(new double[][] {{0.199, 0.801}, {0.499, 0.501}, {0.399, 0.601},
                    {0.799, 0.201}, {0.899, 0.101}});

    Map<Double, Double> expTPR = new HashMap<>();
    double totalPositives = 2.0;
    expTPR.put(0.0, 2.0 / totalPositives); //All predicted class 1 -> 2 true positives / 2 total positives
    expTPR.put(0.1, 2.0 / totalPositives);
    expTPR.put(0.2, 2.0 / totalPositives);
    expTPR.put(0.3, 2.0 / totalPositives);
    expTPR.put(0.4, 2.0 / totalPositives);
    expTPR.put(0.5, 2.0 / totalPositives);
    expTPR.put(0.6, 1.0 / totalPositives); //At threshold of 0.6, only 1 of 2 positives are predicted positive
    expTPR.put(0.7, 1.0 / totalPositives);
    expTPR.put(0.8, 1.0 / totalPositives);
    expTPR.put(0.9, 0.0 / totalPositives); //At threshold of 0.9, 0 of 2 positives are predicted positive
    expTPR.put(1.0, 0.0 / totalPositives);

    Map<Double, Double> expFPR = new HashMap<>();
    double totalNegatives = 3.0;
    expFPR.put(0.0, 3.0 / totalNegatives); //All predicted class 1 -> 3 false positives / 3 total negatives
    expFPR.put(0.1, 3.0 / totalNegatives);
    expFPR.put(0.2, 2.0 / totalNegatives); //At threshold of 0.2: 1 true negative, 2 false positives
    expFPR.put(0.3, 1.0 / totalNegatives); //At threshold of 0.3: 2 true negative, 1 false positive
    expFPR.put(0.4, 1.0 / totalNegatives);
    expFPR.put(0.5, 1.0 / totalNegatives);
    expFPR.put(0.6, 1.0 / totalNegatives);
    expFPR.put(0.7, 0.0 / totalNegatives); //At threshold of 0.7: 3 true negatives, 0 false positives
    expFPR.put(0.8, 0.0 / totalNegatives);
    expFPR.put(0.9, 0.0 / totalNegatives);
    expFPR.put(1.0, 0.0 / totalNegatives);

    int[] expTPs = new int[] {2, 2, 2, 2, 2, 2, 1, 1, 1, 0, 0};
    int[] expFPs = new int[] {3, 3, 2, 1, 1, 1, 1, 0, 0, 0, 0};
    int[] expFNs = new int[11];
    int[] expTNs = new int[11];
    for (int i = 0; i < 11; i++) {
        expFNs[i] = (int) totalPositives - expTPs[i];
        expTNs[i] = 5 - expTPs[i] - expFPs[i] - expFNs[i];
    }

    ROC roc = new ROC(10);
    roc.eval(labels, prediction);

    RocCurve rocCurve = roc.getRocCurve();

    assertEquals(11, rocCurve.getThreshold().length);
    assertEquals(11, rocCurve.getFpr().length);
    assertEquals(11, rocCurve.getTpr().length);

    for (int i = 0; i < 11; i++) {
        double expThreshold = i / 10.0;
        assertEquals(expThreshold, rocCurve.getThreshold(i), 1e-5);

        double efpr = expFPR.get(expThreshold);
        double afpr = rocCurve.getFalsePositiveRate(i);
        assertEquals(efpr, afpr, 1e-5);

        double etpr = expTPR.get(expThreshold);
        double atpr = rocCurve.getTruePositiveRate(i);
        assertEquals(etpr, atpr, 1e-5);
    }

    //AUC: expected values are based on plotting the ROC curve and manually calculating the area
    double expAUC = 0.5 * 1.0 / 3.0 + (1 - 1 / 3.0) * 1.0;
    double actAUC = roc.calculateAUC();

    assertEquals(expAUC, actAUC, 1e-6);

    PrecisionRecallCurve prc = roc.getPrecisionRecallCurve();
    for (int i = 0; i < 11; i++) {
        PrecisionRecallCurve.Confusion c = prc.getConfusionMatrixAtThreshold(i * 0.1);
        assertEquals(expTPs[i], c.getTpCount());
        assertEquals(expFPs[i], c.getFpCount());
        assertEquals(expFPs[i], c.getFpCount());
        assertEquals(expTNs[i], c.getTnCount());
    }
}
 

@Test
public void testRocTimeSeriesNoMasking() {
    //Same as first test...

    //2 outputs here - probability distribution over classes (softmax)
    INDArray predictions2d = Nd4j.create(new double[][] {{1.0, 0.001}, //add 0.001 to avoid numerical/rounding issues (float vs. double, etc)
                    {0.899, 0.101}, {0.799, 0.201}, {0.699, 0.301}, {0.599, 0.401}, {0.499, 0.501}, {0.399, 0.601},
                    {0.299, 0.701}, {0.199, 0.801}, {0.099, 0.901}});

    INDArray actual2d = Nd4j.create(new double[][] {{1, 0}, {1, 0}, {1, 0}, {1, 0}, {1, 0}, {0, 1}, {0, 1}, {0, 1},
                    {0, 1}, {0, 1}});

    INDArray predictions3d = Nd4j.create(2, 2, 5);
    INDArray firstTSp =
                    predictions3d.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all()).transpose();
    assertArrayEquals(new long[] {5, 2}, firstTSp.shape());
    firstTSp.assign(predictions2d.get(NDArrayIndex.interval(0, 5), NDArrayIndex.all()));

    INDArray secondTSp =
                    predictions3d.get(NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all()).transpose();
    assertArrayEquals(new long[] {5, 2}, secondTSp.shape());
    secondTSp.assign(predictions2d.get(NDArrayIndex.interval(5, 10), NDArrayIndex.all()));

    INDArray labels3d = Nd4j.create(2, 2, 5);
    INDArray firstTS = labels3d.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all()).transpose();
    assertArrayEquals(new long[] {5, 2}, firstTS.shape());
    firstTS.assign(actual2d.get(NDArrayIndex.interval(0, 5), NDArrayIndex.all()));

    INDArray secondTS = labels3d.get(NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all()).transpose();
    assertArrayEquals(new long[] {5, 2}, secondTS.shape());
    secondTS.assign(actual2d.get(NDArrayIndex.interval(5, 10), NDArrayIndex.all()));

    for (int steps : new int[] {10, 0}) { //0 steps: exact
        //            System.out.println("Steps: " + steps);
        ROC rocExp = new ROC(steps);
        rocExp.eval(actual2d, predictions2d);

        ROC rocAct = new ROC(steps);
        rocAct.evalTimeSeries(labels3d, predictions3d);

        assertEquals(rocExp.calculateAUC(), rocAct.calculateAUC(), 1e-6);
        assertEquals(rocExp.calculateAUCPR(), rocAct.calculateAUCPR(), 1e-6);

        assertEquals(rocExp.getRocCurve(), rocAct.getRocCurve());
    }
}
 

@Test
public void testRocTimeSeriesMasking() {
    //2 outputs here - probability distribution over classes (softmax)
    INDArray predictions2d = Nd4j.create(new double[][] {{1.0, 0.001}, //add 0.001 to avoid numerical/rounding issues (float vs. double, etc)
                    {0.899, 0.101}, {0.799, 0.201}, {0.699, 0.301}, {0.599, 0.401}, {0.499, 0.501}, {0.399, 0.601},
                    {0.299, 0.701}, {0.199, 0.801}, {0.099, 0.901}});

    INDArray actual2d = Nd4j.create(new double[][] {{1, 0}, {1, 0}, {1, 0}, {1, 0}, {1, 0}, {0, 1}, {0, 1}, {0, 1},
                    {0, 1}, {0, 1}});


    //Create time series data... first time series: length 4. Second time series: length 6
    INDArray predictions3d = Nd4j.create(2, 2, 6);
    INDArray tad = predictions3d.tensorAlongDimension(0, 1, 2).transpose();
    tad.get(NDArrayIndex.interval(0, 4), NDArrayIndex.all())
                    .assign(predictions2d.get(NDArrayIndex.interval(0, 4), NDArrayIndex.all()));

    tad = predictions3d.tensorAlongDimension(1, 1, 2).transpose();
    tad.assign(predictions2d.get(NDArrayIndex.interval(4, 10), NDArrayIndex.all()));


    INDArray labels3d = Nd4j.create(2, 2, 6);
    tad = labels3d.tensorAlongDimension(0, 1, 2).transpose();
    tad.get(NDArrayIndex.interval(0, 4), NDArrayIndex.all())
                    .assign(actual2d.get(NDArrayIndex.interval(0, 4), NDArrayIndex.all()));

    tad = labels3d.tensorAlongDimension(1, 1, 2).transpose();
    tad.assign(actual2d.get(NDArrayIndex.interval(4, 10), NDArrayIndex.all()));


    INDArray mask = Nd4j.zeros(2, 6);
    mask.get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 4)).assign(1);
    mask.get(NDArrayIndex.point(1), NDArrayIndex.all()).assign(1);


    for (int steps : new int[] {20, 0}) { //0 steps: exact
        ROC rocExp = new ROC(steps);
        rocExp.eval(actual2d, predictions2d);

        ROC rocAct = new ROC(steps);
        rocAct.evalTimeSeries(labels3d, predictions3d, mask);

        assertEquals(rocExp.calculateAUC(), rocAct.calculateAUC(), 1e-6);

        assertEquals(rocExp.getRocCurve(), rocAct.getRocCurve());
    }
}
 

@Test
public void testROCMerging() {
    int nArrays = 10;
    int minibatch = 64;
    int nROCs = 3;

    for (int steps : new int[] {0, 20}) { //0 steps: exact, 20 steps: thresholded

        Nd4j.getRandom().setSeed(12345);
        Random r = new Random(12345);

        List<ROC> rocList = new ArrayList<>();
        for (int i = 0; i < nROCs; i++) {
            rocList.add(new ROC(steps));
        }

        ROC single = new ROC(steps);
        for (int i = 0; i < nArrays; i++) {
            INDArray p = Nd4j.rand(minibatch, 2);
            p.diviColumnVector(p.sum(1));

            INDArray l = Nd4j.zeros(minibatch, 2);
            for (int j = 0; j < minibatch; j++) {
                l.putScalar(j, r.nextInt(2), 1.0);
            }

            single.eval(l, p);

            ROC other = rocList.get(i % rocList.size());
            other.eval(l, p);
        }

        ROC first = rocList.get(0);
        for (int i = 1; i < nROCs; i++) {
            first.merge(rocList.get(i));
        }

        double singleAUC = single.calculateAUC();
        assertTrue(singleAUC >= 0.0 && singleAUC <= 1.0);
        assertEquals(singleAUC, first.calculateAUC(), 1e-6);

        assertEquals(single.getRocCurve(), first.getRocCurve());
    }
}
 

@Test
public void testROCMerging2() {
    int nArrays = 10;
    int minibatch = 64;
    int exactAllocBlockSize = 10;
    int nROCs = 3;
    int steps = 0;  //Exact

    Nd4j.getRandom().setSeed(12345);
    Random r = new Random(12345);

    List<ROC> rocList = new ArrayList<>();
    for (int i = 0; i < nROCs; i++) {
        rocList.add(new ROC(steps, true, exactAllocBlockSize));
    }

    ROC single = new ROC(steps);
    for (int i = 0; i < nArrays; i++) {
        INDArray p = Nd4j.rand(minibatch, 2);
        p.diviColumnVector(p.sum(1));

        INDArray l = Nd4j.zeros(minibatch, 2);
        for (int j = 0; j < minibatch; j++) {
            l.putScalar(j, r.nextInt(2), 1.0);
        }

        single.eval(l, p);

        ROC other = rocList.get(i % rocList.size());
        other.eval(l, p);
    }

    ROC first = rocList.get(0);
    for (int i = 1; i < nROCs; i++) {
        first.merge(rocList.get(i));
    }

    double singleAUC = single.calculateAUC();
    assertTrue(singleAUC >= 0.0 && singleAUC <= 1.0);
    assertEquals(singleAUC, first.calculateAUC(), 1e-6);

    assertEquals(single.getRocCurve(), first.getRocCurve());
}
 

@Test
public void testRocMerge(){
    Nd4j.getRandom().setSeed(12345);

    ROC roc = new ROC();
    ROC roc1 = new ROC();
    ROC roc2 = new ROC();

    int nOut = 2;

    Random r = new Random(12345);
    for( int i=0; i<10; i++ ){
        INDArray labels = Nd4j.zeros(3, nOut);
        for( int j=0; j<3; j++ ){
            labels.putScalar(j, r.nextInt(nOut), 1.0 );
        }
        INDArray out = Nd4j.rand(3, nOut);
        out.diviColumnVector(out.sum(1));

        roc.eval(labels, out);
        if(i % 2 == 0){
            roc1.eval(labels, out);
        } else {
            roc2.eval(labels, out);
        }
    }

    roc1.calculateAUC();
    roc1.calculateAUCPR();
    roc2.calculateAUC();
    roc2.calculateAUCPR();

    roc1.merge(roc2);

    double aucExp = roc.calculateAUC();
    double auprc = roc.calculateAUCPR();

    double aucAct = roc1.calculateAUC();
    double auprcAct = roc1.calculateAUCPR();

    assertEquals(aucExp, aucAct, 1e-6);
    assertEquals(auprc, auprcAct, 1e-6);
}
 

@Test
public void testSegmentationBinary(){
    for( int c : new int[]{4, 1}) { //c=1 should be treated as binary classification case
        Nd4j.getRandom().setSeed(12345);
        int mb = 3;
        int h = 3;
        int w = 2;

        //NCHW
        INDArray labels = Nd4j.create(DataType.FLOAT, mb, c, h, w);
        Nd4j.exec(new BernoulliDistribution(labels, 0.5));

        INDArray predictions = Nd4j.rand(DataType.FLOAT, mb, c, h, w);

        ROCBinary e2d = new ROCBinary();
        ROCBinary e4d = new ROCBinary();

        ROC r2d = new ROC();
        e4d.eval(labels, predictions);

        for (int i = 0; i < mb; i++) {
            for (int j = 0; j < h; j++) {
                for (int k = 0; k < w; k++) {
                    INDArray rowLabel = labels.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(j), NDArrayIndex.point(k));
                    INDArray rowPredictions = predictions.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(j), NDArrayIndex.point(k));
                    rowLabel = rowLabel.reshape(1, rowLabel.length());
                    rowPredictions = rowPredictions.reshape(1, rowLabel.length());

                    e2d.eval(rowLabel, rowPredictions);
                    if(c == 1){
                        r2d.eval(rowLabel, rowPredictions);
                    }
                }
            }
        }

        assertEquals(e2d, e4d);

        if(c == 1){
            ROC r4d = new ROC();
            r4d.eval(labels, predictions);
            assertEquals(r2d, r4d);
        }


        //NHWC, etc
        INDArray lOrig = labels;
        INDArray fOrig = predictions;
        for (int i = 0; i < 4; i++) {
            switch (i) {
                case 0:
                    //CNHW - Never really used
                    labels = lOrig.permute(1, 0, 2, 3).dup();
                    predictions = fOrig.permute(1, 0, 2, 3).dup();
                    break;
                case 1:
                    //NCHW
                    labels = lOrig;
                    predictions = fOrig;
                    break;
                case 2:
                    //NHCW - Never really used...
                    labels = lOrig.permute(0, 2, 1, 3).dup();
                    predictions = fOrig.permute(0, 2, 1, 3).dup();
                    break;
                case 3:
                    //NHWC
                    labels = lOrig.permute(0, 2, 3, 1).dup();
                    predictions = fOrig.permute(0, 2, 3, 1).dup();
                    break;
                default:
                    throw new RuntimeException();
            }

            ROCBinary e = new ROCBinary();
            e.setAxis(i);

            e.eval(labels, predictions);
            assertEquals(e2d, e);

            if(c == 1){
                ROC r2 = new ROC();
                r2.setAxis(i);
                r2.eval(labels, predictions);
                assertEquals(r2d, r2);
            }
        }
    }
}
 

@Test
public void testROC() {

    int nArrays = 100;
    int minibatch = 64;
    int steps = 20;
    int nIn = 5;
    int nOut = 2;
    int layerSize = 10;

    MultiLayerConfiguration conf =
                    new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list()
                                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).build())
                                    .layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut)
                                                    .activation(Activation.SOFTMAX).lossFunction(
                                                                    LossFunctions.LossFunction.MCXENT)
                                                    .build())
                                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();


    Nd4j.getRandom().setSeed(12345);
    Random r = new Random(12345);

    ROC local = new ROC(steps);
    List<DataSet> dsList = new ArrayList<>();
    for (int i = 0; i < nArrays; i++) {
        INDArray features = Nd4j.rand(minibatch, nIn);

        INDArray p = net.output(features);

        INDArray l = Nd4j.zeros(minibatch, 2);
        for (int j = 0; j < minibatch; j++) {
            l.putScalar(j, r.nextInt(2), 1.0);
        }

        local.eval(l, p);

        dsList.add(new DataSet(features, l));
    }


    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, net, null);
    JavaRDD<DataSet> rdd = sc.parallelize(dsList);

    ROC sparkROC = sparkNet.evaluateROC(rdd, steps, 32);

    assertEquals(sparkROC.calculateAUC(), sparkROC.calculateAUC(), 1e-6);

    assertEquals(local.getRocCurve(), sparkROC.getRocCurve());
}
 

@Test
public void rocExactEdgeCaseReallocation() {

    //Set reallocation block size to say 20, but then evaluate a 100-length array

    ROC roc = new ROC(0, true, 50);

    roc.eval(Nd4j.rand(100, 1), Nd4j.ones(100, 1));

}