Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#diviColumnVector()

@Test
public void testClipByNorm(){
    //Expected: if array.norm2(1) is less than 1.0, not modified
    //Otherwise: array.tad(x,1) = array.tad(x,1) * 1.0 / array.tad(x,1).norm2()

    Nd4j.getRandom().setSeed(12345);
    INDArray arr = Nd4j.rand(3,5);
    INDArray norm2_1 = arr.norm2(1);
    arr.diviColumnVector(norm2_1);

    norm2_1 = arr.norm2(1);
    assertEquals(Nd4j.ones(3), norm2_1);

    INDArray scale = Nd4j.create(new double[]{1.1, 1.0, 0.9}, new int[]{3});
    arr.muliColumnVector(scale);
    norm2_1 = arr.norm2(1);

    INDArray out = Nd4j.create(arr.shape());

    Nd4j.getExecutioner().exec(DynamicCustomOp.builder("clipbynorm")
            .addInputs(arr)
            .addOutputs(out)
            .addIntegerArguments(1)
            .addFloatingPointArguments(1.0)
            .build());

    INDArray norm2_1b = out.norm2(1);
    INDArray exp = Nd4j.create(new double[]{1.0, 1.0, norm2_1.getDouble(2)}, new int[]{3});

    assertEquals(exp, norm2_1b);
}
 

@Override
public MathMatrix divideColumnVector(MathVector vector) {
    if (vector instanceof Nd4jVector) {
        Nd4jEnvironmentThread thread = EnvironmentThread.getThread(Nd4jEnvironmentThread.class);
        try (MemoryWorkspace workspace = thread.getSpace()) {
            INDArray thisArray = this.getArray();
            INDArray thatArray = Nd4jVector.class.cast(vector).getArray();
            thisArray.diviColumnVector(thatArray);
            return this;
        }
    } else {
        return MathMatrix.super.divideColumnVector(vector);
    }
}
 

@Test
public void testDebugEdgeCase2(){
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    INDArray l1 = Nd4j.create(new double[]{-0.2585039112684677,-0.005179485353710878,0.4348343401770497,0.020356532375728764,-0.1970793298488186});
    INDArray l2 = Nd4j.create(2,l1.size(1));

    INDArray p1 = Nd4j.create(new double[]{1.3979850406519119,0.6169451410155852,1.128993957530918,0.21000426084450596,0.3171215178932696});
    INDArray p2 = Nd4j.create(2, p1.size(1));

    for( int i=0; i<2; i++ ){
        l2.putRow(i, l1);
        p2.putRow(i, p1);
    }

    INDArray norm2_1 = l1.norm2(1);
    INDArray temp1 = p1.mul(l1);
    INDArray out1 = temp1.diviColumnVector(norm2_1);

    INDArray norm2_2 = l2.norm2(1);
    INDArray temp2 = p2.mul(l2);
    INDArray out2 = temp2.diviColumnVector(norm2_2);

    System.out.println("norm2_1: " + Arrays.toString(norm2_1.data().asDouble()));
    System.out.println("norm2_2: " + Arrays.toString(norm2_2.data().asDouble()));

    System.out.println("temp1: " + Arrays.toString(temp1.data().asDouble()));
    System.out.println("temp2: " + Arrays.toString(temp2.data().asDouble()));

    //Outputs here should be identical:
    System.out.println(Arrays.toString(out1.data().asDouble()));
    System.out.println(Arrays.toString(out2.getRow(0).dup().data().asDouble()));
}
 

@Test
public void testCosineDistance() {
    SameDiff sd = SameDiff.create();

    int nOut = 4;
    int minibatch = 10;
    SDVariable predictions = sd.var("in", DataType.DOUBLE, minibatch, nOut);
    SDVariable labels = sd.var("labels", DataType.DOUBLE, -1, nOut);

    INDArray wArr = Nd4j.create(new double[][]{
            {0, 0, 0, 0}, {0, 0, 1, 1}, {1, 1, 0, 0}, {1, 1, 1, 1}, {1, 1, 1, 1},
            {2, 2, 2, 2}, {2, 2, 2, 2}, {2, 2, 2, 2}, {2, 2, 2, 2}, {2, 2, 2, 2}});
    SDVariable w = sd.var("weights", wArr);

    LossReduce reduction = LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT;

    INDArray predictionsArr = Nd4j.randn(DataType.DOUBLE, minibatch, nOut);
    INDArray labelsArr = Nd4j.randn(DataType.DOUBLE, minibatch, nOut);

    predictionsArr.diviColumnVector(predictionsArr.norm2(1));
    labelsArr.diviColumnVector(labelsArr.norm2(1));

    SDVariable loss = sd.loss().cosineDistance("loss", labels, predictions, w, reduction, 0);
    SDVariable loss2 = sd.loss().cosineDistance("loss2", labels, predictions, null, reduction, 0);
    sd.associateArrayWithVariable(predictionsArr, predictions);
    sd.associateArrayWithVariable(labelsArr, labels);

    INDArray y_exp = loss.eval();
    INDArray y_exp2 = loss2.eval();

    INDArray y = Nd4j.loss().cosineDistance(labelsArr, predictionsArr, wArr, reduction, 0);
    INDArray y2 = Nd4j.loss().cosineDistance(labelsArr, predictionsArr, null, reduction, 0);
    assertEquals(y_exp, y);
    assertEquals(y_exp2, y2);
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    INDArray yhat = activationFn.getActivation(preOutput.dup(), true);
    INDArray yL2norm = labels.norm2(1);

    INDArray yhatL2norm = yhat.norm2(1);
    INDArray yhatL2normSq = yhatL2norm.mul(yhatL2norm);

    //Note: This is not really the L1 norm since I am not taking abs values
    INDArray yhatDotyL1norm = labels.mul(yhat).sum(true,1);

    INDArray dLda = labels.mulColumnVector(yhatL2normSq);
    dLda.subi(yhat.mulColumnVector(yhatDotyL1norm));

    // transform vals to avoid nans before div
    yL2norm = Transforms.max(yL2norm, Nd4j.EPS_THRESHOLD, false);
    yhatL2norm = Transforms.max(yhatL2norm, Nd4j.EPS_THRESHOLD, false);
    yhatL2normSq = Transforms.max(yhatL2normSq, Nd4j.EPS_THRESHOLD, false);

    dLda.diviColumnVector(yL2norm);
    dLda.diviColumnVector(yhatL2norm.mul(yhatL2normSq));
    dLda.muli(-1);

    //dL/dz
    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO loss functions with params

    if (mask != null) {
        gradients.muliColumnVector(mask);
    }

    return gradients;
}
 

/**
 *
 * @param labels
 * @param preOutput
 * @param activationFn
 * @param mask
 * @return
 */
public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype

    /*
     mean of -(y.dot(yhat)/||y||*||yhat||)
     */
    INDArray postOutput = activationFn.getActivation(preOutput.dup(), true);

    INDArray yhatmag = postOutput.norm2(1);
    INDArray ymag = labels.norm2(1);
    yhatmag = Transforms.max(yhatmag, Nd4j.EPS_THRESHOLD, false);
    ymag = Transforms.max(ymag, Nd4j.EPS_THRESHOLD, false);

    INDArray scoreArr = postOutput.mul(labels);
    scoreArr.diviColumnVector(yhatmag);
    scoreArr.diviColumnVector(ymag);

    if (mask != null) {
        if (!mask.isColumnVector()) {
            //Per-output masking doesn't really make sense for cosine proximity
            throw new UnsupportedOperationException("Expected column vector mask array for LossCosineProximity."
                            + " Got mask array with shape " + Arrays.toString(mask.shape())
                            + "; per-output masking is not " + "supported for LossCosineProximity");
        }
        scoreArr.muliColumnVector(mask);
    }
    return scoreArr.muli(-1);
}
 

@Test
public void testRocBinaryMerge(){
    Nd4j.getRandom().setSeed(12345);

    ROCBinary roc = new ROCBinary();
    ROCBinary roc1 = new ROCBinary();
    ROCBinary roc2 = new ROCBinary();

    int nOut = 5;

    for( int i=0; i<10; i++ ){
        INDArray labels = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(3, nOut),0.5));
        INDArray out = Nd4j.rand(3, nOut);
        out.diviColumnVector(out.sum(1));

        roc.eval(labels, out);
        if(i % 2 == 0){
            roc1.eval(labels, out);
        } else {
            roc2.eval(labels, out);
        }
    }

    for( int i=0; i<nOut; i++ ) {
        roc1.calculateAUC(i);
        roc1.calculateAUCPR(i);
        roc2.calculateAUC(i);
        roc2.calculateAUCPR(i);
    }

    roc1.merge(roc2);

    for( int i=0; i<nOut; i++ ) {

        double aucExp = roc.calculateAUC(i);
        double auprc = roc.calculateAUCPR(i);

        double aucAct = roc1.calculateAUC(i);
        double auprcAct = roc1.calculateAUCPR(i);

        assertEquals(aucExp, aucAct, 1e-6);
        assertEquals(auprc, auprcAct, 1e-6);
    }
}
 

@Test
public void testRocMerge(){
    Nd4j.getRandom().setSeed(12345);

    ROC roc = new ROC();
    ROC roc1 = new ROC();
    ROC roc2 = new ROC();

    int nOut = 2;

    Random r = new Random(12345);
    for( int i=0; i<10; i++ ){
        INDArray labels = Nd4j.zeros(3, nOut);
        for( int j=0; j<3; j++ ){
            labels.putScalar(j, r.nextInt(nOut), 1.0 );
        }
        INDArray out = Nd4j.rand(3, nOut);
        out.diviColumnVector(out.sum(1));

        roc.eval(labels, out);
        if(i % 2 == 0){
            roc1.eval(labels, out);
        } else {
            roc2.eval(labels, out);
        }
    }

    roc1.calculateAUC();
    roc1.calculateAUCPR();
    roc2.calculateAUC();
    roc2.calculateAUCPR();

    roc1.merge(roc2);

    double aucExp = roc.calculateAUC();
    double auprc = roc.calculateAUCPR();

    double aucAct = roc1.calculateAUC();
    double auprcAct = roc1.calculateAUCPR();

    assertEquals(aucExp, aucAct, 1e-6);
    assertEquals(auprc, auprcAct, 1e-6);
}
 

@Test
public void testROCMultiMerging() {

    int nArrays = 10;
    int minibatch = 64;
    int nROCs = 3;
    int nClasses = 3;

    for (int steps : new int[] {20, 0}) { //0 steps: exact
        //            int steps = 20;

        Nd4j.getRandom().setSeed(12345);
        Random r = new Random(12345);

        List<ROCMultiClass> rocList = new ArrayList<>();
        for (int i = 0; i < nROCs; i++) {
            rocList.add(new ROCMultiClass(steps));
        }

        ROCMultiClass single = new ROCMultiClass(steps);
        for (int i = 0; i < nArrays; i++) {
            INDArray p = Nd4j.rand(minibatch, nClasses);
            p.diviColumnVector(p.sum(1));

            INDArray l = Nd4j.zeros(minibatch, nClasses);
            for (int j = 0; j < minibatch; j++) {
                l.putScalar(j, r.nextInt(nClasses), 1.0);
            }

            single.eval(l, p);

            ROCMultiClass other = rocList.get(i % rocList.size());
            other.eval(l, p);
        }

        ROCMultiClass first = rocList.get(0);
        for (int i = 1; i < nROCs; i++) {
            first.merge(rocList.get(i));
        }

        for (int i = 0; i < nClasses; i++) {
            assertEquals(single.calculateAUC(i), first.calculateAUC(i), 1e-6);

            assertEquals(single.getRocCurve(i), first.getRocCurve(i));
        }
    }
}
 

@Test
public void testROCMerging2() {
    int nArrays = 10;
    int minibatch = 64;
    int exactAllocBlockSize = 10;
    int nROCs = 3;
    int steps = 0;  //Exact

    Nd4j.getRandom().setSeed(12345);
    Random r = new Random(12345);

    List<ROC> rocList = new ArrayList<>();
    for (int i = 0; i < nROCs; i++) {
        rocList.add(new ROC(steps, true, exactAllocBlockSize));
    }

    ROC single = new ROC(steps);
    for (int i = 0; i < nArrays; i++) {
        INDArray p = Nd4j.rand(minibatch, 2);
        p.diviColumnVector(p.sum(1));

        INDArray l = Nd4j.zeros(minibatch, 2);
        for (int j = 0; j < minibatch; j++) {
            l.putScalar(j, r.nextInt(2), 1.0);
        }

        single.eval(l, p);

        ROC other = rocList.get(i % rocList.size());
        other.eval(l, p);
    }

    ROC first = rocList.get(0);
    for (int i = 1; i < nROCs; i++) {
        first.merge(rocList.get(i));
    }

    double singleAUC = single.calculateAUC();
    assertTrue(singleAUC >= 0.0 && singleAUC <= 1.0);
    assertEquals(singleAUC, first.calculateAUC(), 1e-6);

    assertEquals(single.getRocCurve(), first.getRocCurve());
}
 

/**
 * This method returns the gradient of the cost function with respect to the
 * output from the previous layer.  For this cost function, the gradient
 * is derived from Bishop's paper "Mixture Density Networks" (1994) which
 * gives an elegant closed-form expression for the derivatives with respect
 * to each of the output components.
 * @param labels Labels to train on.
 * @param preOutput Output of neural network before applying the final activation function.
 * @param activationFn Activation function of output layer.
 * @param mask Mask to apply to gradients.
 * @return Gradient of cost function with respect to preOutput parameters.
 */
@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    long nSamples = labels.size(0);

    INDArray output = activationFn.getActivation(preOutput.dup(), false);

    MixtureDensityComponents mdc = extractComponents(output);

    INDArray gradient = Nd4j.zeros(nSamples, preOutput.columns());

    INDArray labelsMinusMu = labelsMinusMu(labels, mdc.mu);
    INDArray labelsMinusMuSquared = labelsMinusMu.mul(labelsMinusMu).sum(2);

    // This computes pi_i, see Bishop equation (30).
    // See http://www.plsyard.com/dealing-overflow-and-underflow-in-softmax-function/
    // this post for why we calculate the pi_i in this way.
    // With the exponential function here, we have to be very careful
    // about overflow/underflow considerations even with
    // fairly intermediate values.  Subtracting the max
    // here helps to ensure over/underflow does not happen here.
    // This isn't exactly a softmax because there's an 'alpha' coefficient
    // here, but the technique works, nonetheless.
    INDArray variance = mdc.sigma.mul(mdc.sigma);
    INDArray minustwovariance = variance.mul(2).negi();
    INDArray normalPart = mdc.alpha.div(Transforms.pow(mdc.sigma.mul(SQRT_TWO_PI), mLabelWidth));
    INDArray exponent = labelsMinusMuSquared.div(minustwovariance);
    INDArray exponentMax = exponent.max(1);
    exponent.subiColumnVector(exponentMax);
    INDArray pi = Transforms.exp(exponent).muli(normalPart);
    INDArray piDivisor = pi.sum(true,1);
    pi.diviColumnVector(piDivisor);

    // See Bishop equation (35)
    //INDArray dLdZAlpha = Nd4j.zeros(nSamples, nLabelsPerSample, mMixturesPerLabel); //mdc.alpha.sub(pi);
    INDArray dLdZAlpha = mdc.alpha.sub(pi);
    // See Bishop equation (38)
    INDArray dLdZSigma = (labelsMinusMuSquared.div(variance).subi(mLabelWidth)).muli(-1).muli(pi);
    // See Bishop equation (39)

    // This turned out to be way less efficient than
    // the simple 'for' loop here.
    //INDArray dLdZMu = pi
    //        .div(variance)
    //        .reshape(nSamples, mMixtures, 1)
    //        .repeat(2, mLabelWidth)
    //        .muli(labelsMinusMu)
    //        .negi()
    //        .reshape(nSamples, mMixtures * mLabelWidth);

    INDArray dLdZMu = Nd4j.create(nSamples, mMixtures, mLabelWidth);
    for (int k = 0; k < mLabelWidth; k++) {
        dLdZMu.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(k)},
                        labelsMinusMu.get(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(),
                                        NDArrayIndex.point(k)}).muli(pi).divi(variance).negi());
    }
    dLdZMu = dLdZMu.reshape(nSamples, mMixtures * mLabelWidth);

    // Place components of gradient into gradient holder.
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, mMixtures)}, dLdZAlpha);
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(mMixtures, mMixtures * 2)},
                    dLdZSigma);
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(),
                    NDArrayIndex.interval(mMixtures * 2, (mLabelWidth + 2) * mMixtures)}, dLdZMu);

    INDArray gradients = activationFn.backprop(preOutput, gradient).getFirst();

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Test
public void testCompare2Vs3Classes() {

    //ROC multi-class: 2 vs. 3 classes should be the same, if we add two of the classes together...
    //Both methods implement one vs. all ROC/AUC in different ways

    int nExamples = 200;
    INDArray predictions3 = Nd4j.rand(nExamples, 3);
    INDArray tempSum = predictions3.sum(1);
    predictions3.diviColumnVector(tempSum);

    INDArray labels3 = Nd4j.create(nExamples, 3);
    Random r = new Random(12345);
    for (int i = 0; i < nExamples; i++) {
        labels3.putScalar(i, r.nextInt(3), 1.0);
    }

    INDArray predictions2 = Nd4j.zeros(nExamples, 2);
    predictions2.getColumn(0).assign(predictions3.getColumn(0));
    predictions2.getColumn(0).addi(predictions3.getColumn(1));
    predictions2.getColumn(1).addi(predictions3.getColumn(2));

    INDArray labels2 = Nd4j.zeros(nExamples, 2);
    labels2.getColumn(0).assign(labels3.getColumn(0));
    labels2.getColumn(0).addi(labels3.getColumn(1));
    labels2.getColumn(1).addi(labels3.getColumn(2));

    for (int numSteps : new int[] {30, 0}) { //Steps = 0: exact

        ROCMultiClass rocMultiClass3 = new ROCMultiClass(numSteps);
        ROCMultiClass rocMultiClass2 = new ROCMultiClass(numSteps);

        rocMultiClass3.eval(labels3, predictions3);
        rocMultiClass2.eval(labels2, predictions2);

        double auc3 = rocMultiClass3.calculateAUC(2);
        double auc2 = rocMultiClass2.calculateAUC(1);

        assertEquals(auc2, auc3, 1e-6);

        RocCurve c3 = rocMultiClass3.getRocCurve(2);
        RocCurve c2 = rocMultiClass2.getRocCurve(1);

        assertArrayEquals(c2.getThreshold(), c3.getThreshold(), 1e-6);
        assertArrayEquals(c2.getFpr(), c3.getFpr(), 1e-6);
        assertArrayEquals(c2.getTpr(), c3.getTpr(), 1e-6);
    }
}
 

/** Returns tuple: {Gradient,Delta,Output} given preOut */
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    if (labels2d.size(1) != preOut.size(1)) {
        throw new DL4JInvalidInputException(
                        "Labels array numColumns (size(1) = " + labels2d.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOut.size(1) + ") " + layerId());
    }

    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(CenterLossParamInitializer.WEIGHT_KEY);
    INDArray biasGradView = gradientViews.get(CenterLossParamInitializer.BIAS_KEY);
    INDArray centersGradView = gradientViews.get(CenterLossParamInitializer.CENTER_KEY);

    // centers delta
    double alpha = layerConf().getAlpha();

    INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
    INDArray l = labels.castTo(centers.dataType()); //Ensure correct dtype (same as params); no-op if already correct dtype
    INDArray centersForExamples = l.mmul(centers);
    INDArray diff = centersForExamples.sub(input).muli(alpha);
    INDArray numerator = l.transpose().mmul(diff);
    INDArray denominator = l.sum(0).reshape(l.size(1), 1).addi(1.0);

    INDArray deltaC;
    if (layerConf().getGradientCheck()) {
        double lambda = layerConf().getLambda();
        //For gradient checks: need to multiply dLc/dcj by lambda to get dL/dcj
        deltaC = numerator.muli(lambda);
    } else {
        deltaC = numerator.diviColumnVector(denominator);
    }
    centersGradView.assign(deltaC);



    // other standard calculations
    Nd4j.gemm(input, delta, weightGradView, true, false, 1.0, 0.0); //Equivalent to:  weightGradView.assign(input.transpose().mmul(delta));
    delta.sum(biasGradView, 0); //biasGradView is initialized/zeroed first in sum op

    gradient.gradientForVariable().put(CenterLossParamInitializer.WEIGHT_KEY, weightGradView);
    gradient.gradientForVariable().put(CenterLossParamInitializer.BIAS_KEY, biasGradView);
    gradient.gradientForVariable().put(CenterLossParamInitializer.CENTER_KEY, centersGradView);

    return new Pair<>(gradient, delta);
}
 

@Test
public void testNewInstances() {
    boolean print = true;
    Nd4j.getRandom().setSeed(12345);

    Evaluation evaluation = new Evaluation();
    EvaluationBinary evaluationBinary = new EvaluationBinary();
    ROC roc = new ROC(2);
    ROCBinary roc2 = new ROCBinary(2);
    ROCMultiClass roc3 = new ROCMultiClass(2);
    RegressionEvaluation regressionEvaluation = new RegressionEvaluation();
    EvaluationCalibration ec = new EvaluationCalibration();


    IEvaluation[] arr = new IEvaluation[] {evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec};

    INDArray evalLabel1 = Nd4j.create(10, 3);
    for (int i = 0; i < 10; i++) {
        evalLabel1.putScalar(i, i % 3, 1.0);
    }
    INDArray evalProb1 = Nd4j.rand(10, 3);
    evalProb1.diviColumnVector(evalProb1.sum(1));

    evaluation.eval(evalLabel1, evalProb1);
    roc3.eval(evalLabel1, evalProb1);
    ec.eval(evalLabel1, evalProb1);

    INDArray evalLabel2 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5));
    INDArray evalProb2 = Nd4j.rand(10, 3);
    evaluationBinary.eval(evalLabel2, evalProb2);
    roc2.eval(evalLabel2, evalProb2);

    INDArray evalLabel3 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5));
    INDArray evalProb3 = Nd4j.rand(10, 1);
    roc.eval(evalLabel3, evalProb3);

    INDArray reg1 = Nd4j.rand(10, 3);
    INDArray reg2 = Nd4j.rand(10, 3);

    regressionEvaluation.eval(reg1, reg2);

    Evaluation evaluation2 = evaluation.newInstance();
    EvaluationBinary evaluationBinary2 = evaluationBinary.newInstance();
    ROC roc_2 = roc.newInstance();
    ROCBinary roc22 = roc2.newInstance();
    ROCMultiClass roc32 = roc3.newInstance();
    RegressionEvaluation regressionEvaluation2 = regressionEvaluation.newInstance();
    EvaluationCalibration ec2 = ec.newInstance();

    IEvaluation[] arr2 = new IEvaluation[] {evaluation2, evaluationBinary2, roc_2, roc22, roc32, regressionEvaluation2, ec2};

    evaluation2.eval(evalLabel1, evalProb1);
    roc32.eval(evalLabel1, evalProb1);
    ec2.eval(evalLabel1, evalProb1);

    evaluationBinary2.eval(evalLabel2, evalProb2);
    roc22.eval(evalLabel2, evalProb2);

    roc_2.eval(evalLabel3, evalProb3);

    regressionEvaluation2.eval(reg1, reg2);

    for (int i = 0 ; i < arr.length ; i++) {

        IEvaluation e = arr[i];
        IEvaluation e2 = arr2[i];
        assertEquals("Json not equal ", e.toJson(), e2.toJson());
        assertEquals("Stats not equal ", e.stats(), e2.stats());
    }
}
 

@Test
public void testJsonYamlCurves() {
    ROC roc = new ROC(0);

    INDArray evalLabel =
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 1), 0.5));
    INDArray evalProb = Nd4j.rand(100, 1);
    roc.eval(evalLabel, evalProb);

    RocCurve c = roc.getRocCurve();
    PrecisionRecallCurve prc = roc.getPrecisionRecallCurve();

    String json1 = c.toJson();
    String json2 = prc.toJson();

    RocCurve c2 = RocCurve.fromJson(json1);
    PrecisionRecallCurve prc2 = PrecisionRecallCurve.fromJson(json2);

    assertEquals(c, c2);
    assertEquals(prc, prc2);

    //        System.out.println(json1);

    //Also test: histograms

    EvaluationCalibration ec = new EvaluationCalibration();

    evalLabel = Nd4j.create(10, 3);
    for (int i = 0; i < 10; i++) {
        evalLabel.putScalar(i, i % 3, 1.0);
    }
    evalProb = Nd4j.rand(10, 3);
    evalProb.diviColumnVector(evalProb.sum(1));
    ec.eval(evalLabel, evalProb);

    Histogram[] histograms = new Histogram[] {ec.getResidualPlotAllClasses(), ec.getResidualPlot(0),
                    ec.getResidualPlot(1), ec.getProbabilityHistogramAllClasses(), ec.getProbabilityHistogram(0),
                    ec.getProbabilityHistogram(1)};

    for (Histogram h : histograms) {
        String json = h.toJson();
        String yaml = h.toYaml();

        Histogram h2 = Histogram.fromJson(json);
        Histogram h3 = Histogram.fromYaml(yaml);

        assertEquals(h, h2);
        assertEquals(h2, h3);
    }

}
 

@Test
public void testSerde() {
    boolean print = false;
    Nd4j.getRandom().setSeed(12345);

    Evaluation evaluation = new Evaluation();
    EvaluationBinary evaluationBinary = new EvaluationBinary();
    ROC roc = new ROC(2);
    ROCBinary roc2 = new ROCBinary(2);
    ROCMultiClass roc3 = new ROCMultiClass(2);
    RegressionEvaluation regressionEvaluation = new RegressionEvaluation();
    EvaluationCalibration ec = new EvaluationCalibration();


    IEvaluation[] arr = new IEvaluation[] {evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec};

    INDArray evalLabel = Nd4j.create(10, 3);
    for (int i = 0; i < 10; i++) {
        evalLabel.putScalar(i, i % 3, 1.0);
    }
    INDArray evalProb = Nd4j.rand(10, 3);
    evalProb.diviColumnVector(evalProb.sum(1));
    evaluation.eval(evalLabel, evalProb);
    roc3.eval(evalLabel, evalProb);
    ec.eval(evalLabel, evalProb);

    evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5));
    evalProb = Nd4j.rand(10, 3);
    evaluationBinary.eval(evalLabel, evalProb);
    roc2.eval(evalLabel, evalProb);

    evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5));
    evalProb = Nd4j.rand(10, 1);
    roc.eval(evalLabel, evalProb);

    regressionEvaluation.eval(Nd4j.rand(10, 3), Nd4j.rand(10, 3));



    for (IEvaluation e : arr) {
        String json = e.toJson();
        if (print) {
            System.out.println(e.getClass() + "\n" + json + "\n\n");
        }

        IEvaluation fromJson = BaseEvaluation.fromJson(json, BaseEvaluation.class);
        assertEquals(e.toJson(), fromJson.toJson());
    }
}
 

@Test
public void testSerde() {
    boolean print = false;
    Nd4j.getRandom().setSeed(12345);

    Evaluation evaluation = new Evaluation();
    EvaluationBinary evaluationBinary = new EvaluationBinary();
    ROC roc = new ROC(2);
    ROCBinary roc2 = new ROCBinary(2);
    ROCMultiClass roc3 = new ROCMultiClass(2);
    RegressionEvaluation regressionEvaluation = new RegressionEvaluation();
    EvaluationCalibration ec = new EvaluationCalibration();


    org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] {evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec};

    INDArray evalLabel = Nd4j.create(10, 3);
    for (int i = 0; i < 10; i++) {
        evalLabel.putScalar(i, i % 3, 1.0);
    }
    INDArray evalProb = Nd4j.rand(10, 3);
    evalProb.diviColumnVector(evalProb.sum(true,1));
    evaluation.eval(evalLabel, evalProb);
    roc3.eval(evalLabel, evalProb);
    ec.eval(evalLabel, evalProb);

    evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5));
    evalProb = Nd4j.rand(10, 3);
    evaluationBinary.eval(evalLabel, evalProb);
    roc2.eval(evalLabel, evalProb);

    evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5));
    evalProb = Nd4j.rand(10, 1);
    roc.eval(evalLabel, evalProb);

    regressionEvaluation.eval(Nd4j.rand(10, 3), Nd4j.rand(10, 3));



    for (org.nd4j.evaluation.IEvaluation e : arr) {
        String json = e.toJson();
        if (print) {
            System.out.println(e.getClass() + "\n" + json + "\n\n");
        }

        IEvaluation fromJson = (IEvaluation) BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class);
        assertEquals(e.toJson(), fromJson.toJson());
    }
}
 

public static INDArray maskedPoolingConvolution(PoolingType poolingType, INDArray toReduce, INDArray mask, int pnorm, DataType dataType) {
    if(mask.rank() != 4){
        //TODO BETTER ERROR MESSAGE EXPLAINING FORMAT
        //TODO ALSO HANDLE LEGACY FORMAT WITH WARNING WHERE POSSIBLE
        throw new IllegalStateException("Expected rank 4 mask array: Got array with shape " + Arrays.toString(mask.shape()));
    }

    mask = mask.castTo(dataType);   //no-op if already correct dtype

    // [minibatch, channels, h, w] data with a mask array of shape [minibatch, 1, X, Y]
    // where X=(1 or inH) and Y=(1 or inW)

    //General case: must be equal or 1 on each dimension
    int[] dimensions = new int[4];
    int count = 0;
    for(int i=0; i<4; i++ ){
        if(toReduce.size(i) == mask.size(i)){
            dimensions[count++] = i;
        }
    }
    if(count < 4){
        dimensions = Arrays.copyOfRange(dimensions, 0, count);
    }

    switch (poolingType) {
        case MAX:
            //TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
            INDArray negInfMask;
            if(mask.dataType() == DataType.BOOL){
                negInfMask = Transforms.not(mask).castTo(dataType);
            } else {
                negInfMask = mask.rsub(1.0);
            }
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, dimensions));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            return withInf.max(2, 3);
        case AVG:
        case SUM:
            INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, dimensions));

            INDArray summed = masked.sum(2, 3);
            if (poolingType == PoolingType.SUM) {
                return summed;
            }
            INDArray maskCounts = mask.sum(1,2,3);
            summed.diviColumnVector(maskCounts);
            return summed;

        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, dimensions));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(2, 3);

            return Transforms.pow(pNorm, 1.0 / pnorm);
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
 

public static INDArray maskedPoolingTimeSeries(PoolingType poolingType, INDArray toReduce, INDArray mask,
                int pnorm, DataType dataType) {
    if (toReduce.rank() != 3) {
        throw new IllegalArgumentException("Expect rank 3 array: got " + toReduce.rank());
    }
    if (mask.rank() != 2) {
        throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
    }

    toReduce = toReduce.castTo(dataType);
    mask = mask.castTo(dataType);

    //Sum pooling: easy. Multiply by mask, then sum as normal
    //Average pooling: as above, but do a broadcast element-wise divi by mask.sum(1)
    //Max pooling: set to -inf if mask is 0, then do max as normal

    switch (poolingType) {
        case MAX:
            INDArray negInfMask = mask.castTo(dataType).rsub(1.0);
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, 0, 2));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            return withInf.max(2);
        case AVG:
        case SUM:
            INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, 0, 2));
            INDArray summed = masked.sum(2);
            if (poolingType == PoolingType.SUM) {
                return summed;
            }

            INDArray maskCounts = mask.sum(1);
            summed.diviColumnVector(maskCounts);
            return summed;
        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, 0, 2));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(2);

            return Transforms.pow(pNorm, 1.0 / pnorm);
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
 

@Test
    public void testDebugEdgeCase2(){
        DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
        INDArray l1 = Nd4j.create(new double[]{-0.2585039112684677,-0.005179485353710878,0.4348343401770497,0.020356532375728764,-0.1970793298488186});
        INDArray l2 = Nd4j.create(2,l1.size(1));

        INDArray p1 = Nd4j.create(new double[]{1.3979850406519119,0.6169451410155852,1.128993957530918,0.21000426084450596,0.3171215178932696});
        INDArray p2 = Nd4j.create(2, p1.size(1));

        for( int i=0; i<2; i++ ){
            l2.putRow(i, l1);
            p2.putRow(i, p1);
        }

        INDArray norm2_1 = l1.norm2(1);
        System.out.println("Queue: " + ((CudaGridExecutioner) Nd4j.getExecutioner()).getQueueLength());

        INDArray temp1 = p1.mul(l1);

        System.out.println("Queue: " + ((CudaGridExecutioner) Nd4j.getExecutioner()).getQueueLength());

//        if (Nd4j.getExecutioner() instanceof CudaGridExecutioner)
//            ((CudaGridExecutioner) Nd4j.getExecutioner()).flushQueueBlocking();

        INDArray out1 = temp1.diviColumnVector(norm2_1);
        System.out.println("------");

        Nd4j.getExecutioner().commit();

        INDArray norm2_2 = l2.norm2(1);

        System.out.println("norm2_1: " + Arrays.toString(norm2_1.data().asDouble()));
        System.out.println("norm2_2: " + Arrays.toString(norm2_2.data().asDouble()));

        INDArray temp2 = p2.mul(l2);



        System.out.println("temp1: " + Arrays.toString(temp1.data().asDouble()));
        System.out.println("temp2: " + Arrays.toString(temp2.data().asDouble()));

        INDArray out2 = temp2.diviColumnVector(norm2_2);

        //Outputs here should be identical:
        System.out.println(Arrays.toString(out1.data().asDouble()));
        System.out.println(Arrays.toString(out2.getRow(0).dup().data().asDouble()));
    }