Java Code Examples for org.nd4j.linalg.ops.transforms.Transforms#abs()

public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    INDArray scoreArr;
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    scoreArr = output.rsubi(labels).divi(labels);
    Transforms.abs(scoreArr, false);
    scoreArr.muli(100.0 / labels.size(1));

    //Weighted loss function
    if (weights != null) {
        if (weights.length() != output.size(1)) {
            throw new IllegalStateException("Weights vector (length " + weights.length()
                            + ") does not match output.size(1)=" + output.size(1));
        }
        scoreArr.muliRowVector(weights.castTo(scoreArr.dataType()));
    }

    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

private static INDArray relError(@NonNull INDArray a1, @NonNull INDArray a2, double minAbsError){
    long numNaN1 = Nd4j.getExecutioner().exec(new MatchCondition(a1, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0);
    long numNaN2 = Nd4j.getExecutioner().exec(new MatchCondition(a2, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0);
    Preconditions.checkState(numNaN1 == 0, "Array 1 has NaNs");
    Preconditions.checkState(numNaN2 == 0, "Array 2 has NaNs");

    INDArray abs1 = Transforms.abs(a1, true);
    INDArray abs2 = Transforms.abs(a2, true);
    INDArray absDiff = Transforms.abs(a1.sub(a2), false);

    //abs(a1-a2) < minAbsError ? 1 : 0
    INDArray greaterThanMinAbs = Transforms.abs(a1.sub(a2), false);
    BooleanIndexing.replaceWhere(greaterThanMinAbs, 0.0, Conditions.lessThan(minAbsError));
    BooleanIndexing.replaceWhere(greaterThanMinAbs, 1.0, Conditions.greaterThan(0.0));

    INDArray result = absDiff.divi(abs1.add(abs2));
    //Only way to have NaNs given there weren't any in original : both 0s
    BooleanIndexing.replaceWhere(result, 0.0, Conditions.isNan());
    //Finally, set to 0 if less than min abs error, or unchanged otherwise
    result.muli(greaterThanMinAbs);

    return result;
}
 

public static void applyAdaMaxUpdater(INDArray gradient, INDArray m, INDArray v, double learningRate, double beta1, double beta2,
                                    double epsilon, int iteration){

    //m = B_1 * m + (1-B_1)*grad
    m.muli(beta1).addi(gradient.mul(1 - beta1));

    //u = max(B_2 * u, |grad|)
    v.muli(beta2);
    Transforms.abs(gradient, false); //In-place should be OK here, original gradient values aren't used again later
    Nd4j.getExecutioner().exec(new Max(v, gradient, v));

    double beta1t = FastMath.pow(beta1, iteration + 1);

    double alphat = learningRate / (1.0 - beta1t);
    if (Double.isNaN(alphat) || Double.isInfinite(alphat) || alphat == 0.0) {
        alphat = epsilon;
    }

    v.addi(1e-32); // prevent NaNs in params
    gradient.assign(m).muli(alphat).divi(v);
}
 

public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    INDArray scoreArr;
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    scoreArr = output.subi(labels);
    Transforms.abs(scoreArr, false);

    //Weighted loss function
    if (weights != null) {
        if (weights.length() != output.size(1)) {
            throw new IllegalStateException("Weights vector (length " + weights.length()
                            + ") does not match output.size(1)=" + output.size(1));
        }
        scoreArr.muliRowVector(weights.castTo(scoreArr.dataType()));
    }

    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    INDArray scoreArr;
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    INDArray yMinusyHat = Transforms.abs(labels.sub(output));
    scoreArr = yMinusyHat.mul(yMinusyHat);
    scoreArr = scoreArr.mul(maskData);

    if (mask != null) {
        scoreArr.muliColumnVector(mask);
    }
    return scoreArr;
}
 

@Test
public void testUnderOverflow() {
    // This dataset will be basically constant with a small std deviation
    // And the constant is large. Checking if algorithm can handle
    double tolerancePerc = 1; //Within 1 %
    double toleranceAbs = 0.0005;
    int nSamples = 1000;
    int bSize = 10;
    int x = -1000000, y = 1000000;
    double z = 1000000;

    INDArray featureX = Nd4j.rand(nSamples, 1).mul(1).add(x);
    INDArray featureY = Nd4j.rand(nSamples, 1).mul(2).add(y);
    INDArray featureZ = Nd4j.rand(nSamples, 1).mul(3).add(z);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);

    INDArray theoreticalMean = Nd4j.create(new float[] {x, y, (float) z}).castTo(Nd4j.defaultFloatingPointType()).reshape(1, -1);

    NormalizerStandardize myNormalizer = new NormalizerStandardize();
    myNormalizer.fit(sampleIter);

    INDArray meanDelta = Transforms.abs(theoreticalMean.sub(myNormalizer.getMean()));
    INDArray meanDeltaPerc = meanDelta.mul(100).div(theoreticalMean);
    assertTrue(meanDeltaPerc.max(1).getDouble(0) < tolerancePerc);

    //this just has to not barf
    //myNormalizer.transform(sampleIter);
    myNormalizer.transform(sampleDataSet);
}
 

@Test
public void testUnderOverflow() {
    // This dataset will be basically constant with a small std deviation
    // And the constant is large. Checking if algorithm can handle
    double tolerancePerc = 1; //Within 1 %
    double toleranceAbs = 0.0005;
    int nSamples = 1000;
    int bSize = 10;
    int x = -1000000, y = 1000000;
    double z = 1000000;

    INDArray featureX = Nd4j.rand(nSamples, 1).mul(1).add(x);
    INDArray featureY = Nd4j.rand(nSamples, 1).mul(2).add(y);
    INDArray featureZ = Nd4j.rand(nSamples, 1).mul(3).add(z);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);

    INDArray theoreticalMean = Nd4j.create(new double[] {x, y, z});

    NormalizerStandardize myNormalizer = new NormalizerStandardize();
    myNormalizer.fit(sampleIter);

    INDArray meanDelta = Transforms.abs(theoreticalMean.sub(myNormalizer.getMean()));
    INDArray meanDeltaPerc = meanDelta.mul(100).div(theoreticalMean);
    assertTrue(meanDeltaPerc.max(1).getDouble(0, 0) < tolerancePerc);

    //this just has to not barf
    //myNormalizer.transform(sampleIter);
    myNormalizer.transform(sampleDataSet);
}
 

private static INDArray relError(@NonNull INDArray a1, @NonNull INDArray a2, double minAbsError){
        long numNaN1 = Nd4j.getExecutioner().exec(new MatchCondition(a1, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0);
        long numNaN2 = Nd4j.getExecutioner().exec(new MatchCondition(a2, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0);
        Preconditions.checkState(numNaN1 == 0, "Array 1 has NaNs");
        Preconditions.checkState(numNaN2 == 0, "Array 2 has NaNs");


//        INDArray isZero1 = a1.eq(0.0);
//        INDArray isZero2 = a2.eq(0.0);
//        INDArray bothZero = isZero1.muli(isZero2);

        INDArray abs1 = Transforms.abs(a1, true);
        INDArray abs2 = Transforms.abs(a2, true);
        INDArray absDiff = Transforms.abs(a1.sub(a2), false);

        //abs(a1-a2) < minAbsError ? 1 : 0
        INDArray greaterThanMinAbs = Transforms.abs(a1.sub(a2), false);
        BooleanIndexing.replaceWhere(greaterThanMinAbs, 0.0, Conditions.lessThan(minAbsError));
        BooleanIndexing.replaceWhere(greaterThanMinAbs, 1.0, Conditions.greaterThan(0.0));

        INDArray result = absDiff.divi(abs1.add(abs2));
        //Only way to have NaNs given there weren't any in original : both 0s
        BooleanIndexing.replaceWhere(result, 0.0, Conditions.isNan());
        //Finally, set to 0 if less than min abs error, or unchanged otherwise
        result.muli(greaterThanMinAbs);

//        double maxRE = result.maxNumber().doubleValue();
//        if(maxRE > t.maxRe){
//            System.out.println();
//        }
        return result;
    }
 

@Override
public double calculateScore(MultiLayerNetwork network) {
    dataSetIterator.reset();

    double losSum = 0.0;
    int exCount = 0;
    while (dataSetIterator.hasNext()) {
        DataSet dataSet = dataSetIterator.next();
        if (dataSet == null) {
            break;
        }
        long nEx = dataSet.getFeatures().size(0);

        INDArray output = network.output(dataSet.getFeatures(), false);
        INDArray labels = dataSet.getLabels();

        INDArray score = Transforms.abs(output.sub(labels));
        score = score.div(labels);

        exCount += nEx;
        losSum += score.sumNumber().doubleValue();
    }

    if (average) {
        return losSum / exCount;
    }
    return losSum;
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    INDArray scoreArr;
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    INDArray yMinusyHat = Transforms.abs(labels.sub(output));
    scoreArr = yMinusyHat.mul(yMinusyHat);
    scoreArr = scoreArr.mul(maskData);

    if (mask != null) {
        scoreArr.muliColumnVector(mask);
    }
    return scoreArr;
}
 

private INDArray epsilonHelperFullArray(INDArray inputArray, INDArray epsilon, int[] poolDim) {

        //Broadcast: occurs on the remaining dimensions, after the pool dimensions have been removed.
        //TODO find a more efficient way to do this
        int[] broadcastDims = new int[inputArray.rank() - poolDim.length];
        int count = 0;
        for (int i = 0; i < inputArray.rank(); i++) {
            if (ArrayUtils.contains(poolDim, i))
                continue;
            broadcastDims[count++] = i;
        }

        switch (poolingType) {
            case MAX:
                INDArray isMax = Nd4j.exec(new IsMax(inputArray, inputArray.ulike(), poolDim))[0];
                return Nd4j.getExecutioner().exec(new BroadcastMulOp(isMax, epsilon, isMax, broadcastDims));
            case AVG:
                //if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
                int n = 1;
                for (int d : poolDim) {
                    n *= inputArray.size(d);
                }
                INDArray ret = inputArray.ulike();
                Nd4j.getExecutioner().exec(new BroadcastCopyOp(ret, epsilon, ret, broadcastDims));
                ret.divi(n);

                return ret;
            case SUM:
                INDArray retSum = inputArray.ulike();
                Nd4j.getExecutioner().exec(new BroadcastCopyOp(retSum, epsilon, retSum, broadcastDims));
                return retSum;
            case PNORM:
                int pnorm = layerConf().getPnorm();

                //First: do forward pass to get pNorm array
                INDArray abs = Transforms.abs(inputArray, true);
                Transforms.pow(abs, pnorm, false);

                INDArray pNorm = Transforms.pow(abs.sum(poolDim), 1.0 / pnorm);

                //dL/dIn = dL/dOut * dOut/dIn
                //dOut/dIn = in .* |in|^(p-2) /  ||in||_p^(p-1), where ||in||_p is the output p-norm

                INDArray numerator;
                if (pnorm == 2) {
                    numerator = inputArray.dup();
                } else {
                    INDArray absp2 = Transforms.pow(Transforms.abs(inputArray, true), pnorm - 2, false);
                    numerator = inputArray.mul(absp2);
                }

                INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
                denom.rdivi(epsilon);
                Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, broadcastDims));

                return numerator;
            default:
                throw new RuntimeException("Unknown or not supported pooling type: " + poolingType + " " + layerId());
        }
    }
 

public static INDArray maskedPoolingTimeSeries(PoolingType poolingType, INDArray toReduce, INDArray mask,
                int pnorm, DataType dataType) {
    if (toReduce.rank() != 3) {
        throw new IllegalArgumentException("Expect rank 3 array: got " + toReduce.rank());
    }
    if (mask.rank() != 2) {
        throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
    }

    toReduce = toReduce.castTo(dataType);
    mask = mask.castTo(dataType);

    //Sum pooling: easy. Multiply by mask, then sum as normal
    //Average pooling: as above, but do a broadcast element-wise divi by mask.sum(1)
    //Max pooling: set to -inf if mask is 0, then do max as normal

    switch (poolingType) {
        case MAX:
            INDArray negInfMask = mask.castTo(dataType).rsub(1.0);
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, 0, 2));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            return withInf.max(2);
        case AVG:
        case SUM:
            INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, 0, 2));
            INDArray summed = masked.sum(2);
            if (poolingType == PoolingType.SUM) {
                return summed;
            }

            INDArray maskCounts = mask.sum(1);
            summed.diviColumnVector(maskCounts);
            return summed;
        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, 0, 2));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(2);

            return Transforms.pow(pNorm, 1.0 / pnorm);
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
 

public static void checkIntermediate(Map<String, INDArray> inputs, String modelName, String baseDir, String modelFileName,
                                         ExecuteWith execType, BiFunction<File,String,SameDiff> loader,
                                         Double maxRelErrorOverride, Double minAbsErrorOverride, File localTestDir, boolean printArraysDebugging) throws IOException {
        Preconditions.checkArgument((maxRelErrorOverride == null) == (minAbsErrorOverride == null), "Both maxRelErrorOverride and minAbsErrorOverride" +
                " must be null or both must be provided");
        Nd4j.EPS_THRESHOLD = 1e-3;
        OpExecOrderListener listener = new OpExecOrderListener();       //Used to collect exec order
        Pair<SameDiff, Map<String,INDArray>> p = getGraphAfterExec(baseDir, modelFileName, modelName, inputs, execType, loader, Collections.singletonList(listener), null, printArraysDebugging);
        SameDiff graph = p.getFirst();
        Map<String,INDArray> sdPredictions = p.getSecond();

        //Collect coverage info about ops
        OpValidation.collectTensorflowImportCoverage(graph);

        if (!execType.equals(ExecuteWith.JUST_PRINT)) {
            int count = 0;
            //Evaluate the nodes in their execution order - this is useful for debugging (as we want the *first* failure
            // to be detected before later failures)
            List<String> varNames = new ArrayList<>();
            Map<String,SameDiffOp> fns = graph.getOps();
            List<String> execOrder = listener.getOpNamesList();
            for(String opName : execOrder){
                String[] outputs = graph.getOutputsForOp(fns.get(opName).getOp());
                Collections.addAll(varNames, outputs);
            }

            for (String varName : varNames) {
                if (!inputs.containsKey(varName)) { //avoiding placeholders
                    INDArray tfValue = intermediateVars(modelName, baseDir, varName, localTestDir);
                    if (tfValue == null) {
                        continue;
                    }
                    log.info("Starting check: variable {}", varName);
                    if (skipNode(modelName, varName)) {
                        log.info("\n\tFORCING no check on " + varName);
                    } else {
                        assertArrayEquals("Shape not equal on node " + varName, tfValue.shape(), graph.getVariable(varName).getShape());
                        INDArray sdVal = sdPredictions.get(varName);
                        if(maxRelErrorOverride != null){
                            INDArray diff = Transforms.abs(tfValue.sub(sdVal), false);
                            INDArray absErrorMask = diff.gte(minAbsErrorOverride);   //value 1 if x[i] > minAbsError; value 0 otherwise. Used to get rid of 1e-30 vs. 1e-29 type failures
                            INDArray sumAbs = Transforms.abs(tfValue, true).addi(Transforms.abs(sdVal, true));
                            BooleanIndexing.replaceWhere(sumAbs, 1.0, Conditions.equals(0.0));  //Can only get 0.0 if both are zeros - need to avoid 0/0=NaN
                            INDArray relError = diff.divi(sumAbs);
                            relError.muli(absErrorMask);

                            int countExceeds = Nd4j.getExecutioner().exec(new MatchCondition(relError, Conditions.greaterThan(maxRelErrorOverride))).getInt(0);

                            double maxRE = -1;
                            //Mainly used for analysis in debugger:
                            DifferentialFunction op = null;
                            String[] opInputs = null;
                            if(countExceeds > 0){
                                maxRE = relError.maxNumber().doubleValue();
                                //Find the op that this variable is produced by
                                op = graph.getVariableOutputOp(varName);
                                opInputs = graph.getInputsForOp(op);
                            }


                            assertEquals( varName + ": " + countExceeds + " values exceed maxRelError=" + maxRelErrorOverride
                                    + " with minAbsError=" + minAbsErrorOverride + "; largest observed relError=" + maxRE, 0, countExceeds);
                        } else {
//                            assertEquals("Value not equal on node " + varName, tfValue, sdVal);
                            if(tfValue.equals(sdVal)){
                                System.out.println("Pass: " + varName);
                            } else {
                                System.out.println("FAIL: " + varName);
                                System.out.println("TF:\n" + tfValue);
                                System.out.println("SD:\n" + sdVal);
                            }

                        }
                        log.info("Values and shapes equal for {}", varName);
                        count++;
                    }

                }
            }

            assertTrue("No intermediate variables were checked", count > 0);
        }

        Nd4j.EPS_THRESHOLD = 1e-5;
    }
 

protected void residualDebugOutputIfRequired(INDArray residual){
    if(!encodingDebugMode)
        return;

    double currThreshold = currentThreshold.get().get();
    String currThresholdStr = format(currThreshold);


    INDArray absResidual = Transforms.abs(residual, true);

    double dAmean = absResidual.meanNumber().doubleValue();
    double dAMax = absResidual.maxNumber().doubleValue();
    double dPc50 = absResidual.percentileNumber(50).doubleValue();
    double dPc95 = absResidual.percentileNumber(95).doubleValue();
    double dPc99 = absResidual.percentileNumber(99).doubleValue();
    double dPc999 = absResidual.percentileNumber(99.9).doubleValue();
    double dPc9999 = absResidual.percentileNumber(99.99).doubleValue();

    String amean = format(dAmean).replace('E', 'e');
    String aMax = format(dAMax).replace('E', 'e');
    String pc50 = format(dPc50).replace('E', 'e');
    String pc95 = format(dPc95).replace('E', 'e');
    String pc99 = format(dPc99).replace('E', 'e');
    String pc999 = format(dPc999).replace('E', 'e');
    String pc9999 = format(dPc9999).replace('E', 'e');

    String ameanThr = format(dAmean / currThreshold).replace('E', 'e');
    String aMaxThr = format(dAMax / currThreshold).replace('E', 'e');
    String pc50Thr = format(dPc50 / currThreshold).replace('E', 'e');
    String pc95Thr = format(dPc95 / currThreshold).replace('E', 'e');
    String pc99Thr = format(dPc99 / currThreshold).replace('E', 'e');
    String pc999Thr = format(dPc999 / currThreshold).replace('E', 'e');
    String pc9999Thr = format(dPc9999 / currThreshold).replace('E', 'e');

    long length = absResidual.length();
    long countAbsGTEThreshold = absResidual.gte(currThreshold).sumNumber().longValue();
    double sparsity = countAbsGTEThreshold / (double)length;
    String sparsityStr = format(sparsity);

    log.info("Encoding debug info, residual vector: length: {}, threshold: {}, count > thr: {}, sparsity: {}, amean: {} ({}x); amax: {} ({}x); 50%: {} ({}x); 95%: {} ({}x}; 99%: {} ({}x);  99.9%: {} ({}x); 99.99%: {} ({}x)",
            length, currThresholdStr, countAbsGTEThreshold, sparsityStr,
            amean, ameanThr, aMax, aMaxThr, pc50, pc50Thr,
            pc95, pc95Thr, pc99, pc99Thr, pc999, pc999Thr, pc9999, pc9999Thr);
}
 

@Override
public INDArray noOp() {
    return Transforms.abs(x());
}
 

@Override
public INDArray noOp() {
    return Transforms.abs(x());
}
 

@Test
public void testTransform() {
    /*Random dataset is generated such that
        AX + B where X is from a normal distribution with mean 0 and std 1
        The mean of above will be B and std A
        Obtained mean and std dev are compared to theoretical
        Transformed values should be the same as X with the same seed.
     */
    long randSeed = 12345;

    int nFeatures = 2;
    int nSamples = 6400;
    int bsize = 8;
    int a = 5;
    int b = 100;
    INDArray sampleMean, sampleStd, sampleMeanDelta, sampleStdDelta, delta, deltaPerc;
    double maxDeltaPerc, sampleMeanSEM;

    genRandomDataSet normData = new genRandomDataSet(nSamples, nFeatures, a, b, randSeed);
    genRandomDataSet expectedData = new genRandomDataSet(nSamples, nFeatures, 1, 0, randSeed);
    genRandomDataSet beforeTransformData = new genRandomDataSet(nSamples, nFeatures, a, b, randSeed);

    NormalizerStandardize myNormalizer = new NormalizerStandardize();
    myNormalizer.fitLabel(true);
    DataSetIterator normIterator = normData.getIter(bsize);
    DataSetIterator expectedIterator = expectedData.getIter(bsize);
    DataSetIterator beforeTransformIterator = beforeTransformData.getIter(bsize);

    myNormalizer.fit(normIterator);

    double tolerancePerc = 0.5; //within 0.5%
    sampleMean = myNormalizer.getMean();
    sampleMeanDelta = Transforms.abs(sampleMean.sub(normData.theoreticalMean));
    assertTrue(sampleMeanDelta.mul(100).div(normData.theoreticalMean).max().getDouble(0) < tolerancePerc);
    //sanity check to see if it's within the theoretical standard error of mean
    sampleMeanSEM = sampleMeanDelta.div(normData.theoreticalSEM).max().getDouble(0);
    assertTrue(String.valueOf(sampleMeanSEM), sampleMeanSEM < 2.6); //99% of the time it should be within this many SEMs

    tolerancePerc = 5; //within 5%
    sampleStd = myNormalizer.getStd();
    sampleStdDelta = Transforms.abs(sampleStd.sub(normData.theoreticalStd));
    assertTrue(sampleStdDelta.div(normData.theoreticalStd).max().mul(100).getDouble(0) < tolerancePerc);

    tolerancePerc = 1; //within 1%
    normIterator.setPreProcessor(myNormalizer);
    while (normIterator.hasNext()) {
        INDArray before = beforeTransformIterator.next().getFeatures();
        DataSet here = normIterator.next();
        assertEquals(here.getFeatures(), here.getLabels()); //bootstrapping existing test on features
        INDArray after = here.getFeatures();
        INDArray expected = expectedIterator.next().getFeatures();
        delta = Transforms.abs(after.sub(expected));
        deltaPerc = delta.div(before.sub(expected));
        deltaPerc.muli(100);
        maxDeltaPerc = deltaPerc.max(0, 1).getDouble(0);
        //System.out.println("=== BEFORE ===");
        //System.out.println(before);
        //System.out.println("=== AFTER ===");
        //System.out.println(after);
        //System.out.println("=== SHOULD BE ===");
        //System.out.println(expected);
        assertTrue(maxDeltaPerc < tolerancePerc);
    }
}
 

@Test
public void testBruteForce() {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
    //X_scaled = X_std * (max - min) + min
    // Dataset features are scaled consecutive natural numbers
    int nSamples = 500;
    int x = 4, y = 2, z = 3;

    INDArray featureX = Nd4j.linspace(1, nSamples, nSamples).reshape(nSamples, 1);
    INDArray featureY = featureX.mul(y);
    INDArray featureZ = featureX.mul(z);
    featureX.muli(x);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);

    //expected min and max
    INDArray theoreticalMin = Nd4j.create(new double[] {x, y, z}, new long[]{1,3});
    INDArray theoreticalMax = Nd4j.create(new double[] {nSamples * x, nSamples * y, nSamples * z}, new long[]{1,3});
    INDArray theoreticalRange = theoreticalMax.sub(theoreticalMin);

    NormalizerMinMaxScaler myNormalizer = new NormalizerMinMaxScaler();
    myNormalizer.fit(sampleDataSet);

    INDArray minDataSet = myNormalizer.getMin();
    INDArray maxDataSet = myNormalizer.getMax();
    INDArray minDiff = minDataSet.sub(theoreticalMin).max();
    INDArray maxDiff = maxDataSet.sub(theoreticalMax).max();
    assertEquals(minDiff.getDouble(0), 0.0, 0.000000001);
    assertEquals(maxDiff.max().getDouble(0), 0.0, 0.000000001);

    // SAME TEST WITH THE ITERATOR
    int bSize = 1;
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);
    myNormalizer.fit(sampleIter);
    minDataSet = myNormalizer.getMin();
    maxDataSet = myNormalizer.getMax();
    assertEquals(minDataSet.sub(theoreticalMin).max(1).getDouble(0), 0.0, 0.000000001);
    assertEquals(maxDataSet.sub(theoreticalMax).max(1).getDouble(0), 0.0, 0.000000001);

    sampleIter.setPreProcessor(myNormalizer);
    INDArray actual, expected, delta;
    int i = 1;
    while (sampleIter.hasNext()) {
        expected = theoreticalMin.mul(i - 1).div(theoreticalRange);
        actual = sampleIter.next().getFeatures();
        delta = Transforms.abs(actual.sub(expected));
        assertTrue(delta.max(1).getDouble(0) < 0.0001);
        i++;
    }

}
 

@Test
public void testBruteForce() {
    /* This test creates a dataset where feature values are multiples of consecutive natural numbers
       The obtained values are compared to the theoretical mean and std dev
     */
    double tolerancePerc = 0.01; // 0.01% of correct value
    int nSamples = 5120;
    int x = 1, y = 2, z = 3;

    INDArray featureX = Nd4j.linspace(1, nSamples, nSamples, DataType.DOUBLE).reshape(nSamples, 1).mul(x);
    INDArray featureY = featureX.mul(y);
    INDArray featureZ = featureX.mul(z);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);

    double meanNaturalNums = (nSamples + 1) / 2.0;
    INDArray theoreticalMean =
                    Nd4j.create(new double[] {meanNaturalNums * x, meanNaturalNums * y, meanNaturalNums * z}).reshape(1, -1);
    double stdNaturalNums = Math.sqrt((nSamples * nSamples - 1) / 12.0);
    INDArray theoreticalStd =
                    Nd4j.create(new double[] {stdNaturalNums * x, stdNaturalNums * y, stdNaturalNums * z}).reshape(1, -1);

    NormalizerStandardize myNormalizer = new NormalizerStandardize();
    myNormalizer.fit(sampleDataSet);

    INDArray meanDelta = Transforms.abs(theoreticalMean.sub(myNormalizer.getMean()));
    INDArray meanDeltaPerc = meanDelta.div(theoreticalMean).mul(100);
    double maxMeanDeltaPerc = meanDeltaPerc.max(1).getDouble(0);
    assertTrue(maxMeanDeltaPerc < tolerancePerc);

    INDArray stdDelta = Transforms.abs(theoreticalStd.sub(myNormalizer.getStd()));
    INDArray stdDeltaPerc = stdDelta.div(theoreticalStd).mul(100);
    double maxStdDeltaPerc = stdDeltaPerc.max(1).getDouble(0);
    assertTrue(maxStdDeltaPerc < tolerancePerc);

    // SAME TEST WITH THE ITERATOR
    int bSize = 10;
    tolerancePerc = 0.1; // 0.1% of correct value
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);
    myNormalizer.fit(sampleIter);

    meanDelta = Transforms.abs(theoreticalMean.sub(myNormalizer.getMean()));
    meanDeltaPerc = meanDelta.div(theoreticalMean).mul(100);
    maxMeanDeltaPerc = meanDeltaPerc.max(1).getDouble(0);
    assertTrue(maxMeanDeltaPerc < tolerancePerc);

    stdDelta = Transforms.abs(theoreticalStd.sub(myNormalizer.getStd()));
    stdDeltaPerc = stdDelta.div(theoreticalStd).mul(100);
    maxStdDeltaPerc = stdDeltaPerc.max(1).getDouble(0);
    assertTrue(maxStdDeltaPerc < tolerancePerc);
}
 

@Test
public void testTransform() {
    /*Random dataset is generated such that
        AX + B where X is from a normal distribution with mean 0 and std 1
        The mean of above will be B and std A
        Obtained mean and std dev are compared to theoretical
        Transformed values should be the same as X with the same seed.
     */
    long randSeed = 12345;

    int nFeatures = 2;
    int nSamples = 6400;
    int bsize = 8;
    int a = 5;
    int b = 100;
    INDArray sampleMean, sampleStd, sampleMeanDelta, sampleStdDelta, delta, deltaPerc;
    double maxDeltaPerc, sampleMeanSEM;

    genRandomDataSet normData = new genRandomDataSet(nSamples, nFeatures, a, b, randSeed);
    DataSet genRandExpected = normData.theoreticalTransform;
    genRandomDataSet expectedData = new genRandomDataSet(nSamples, nFeatures, 1, 0, randSeed);
    genRandomDataSet beforeTransformData = new genRandomDataSet(nSamples, nFeatures, a, b, randSeed);

    NormalizerStandardize myNormalizer = new NormalizerStandardize();
    DataSetIterator normIterator = normData.getIter(bsize);
    DataSetIterator genRandExpectedIter = new TestDataSetIterator(genRandExpected, bsize);
    DataSetIterator expectedIterator = expectedData.getIter(bsize);
    DataSetIterator beforeTransformIterator = beforeTransformData.getIter(bsize);

    myNormalizer.fit(normIterator);

    double tolerancePerc = 0.10; //within 0.1%
    sampleMean = myNormalizer.getMean();
    sampleMeanDelta = Transforms.abs(sampleMean.sub(normData.theoreticalMean));
    assertTrue(sampleMeanDelta.mul(100).div(normData.theoreticalMean).max().getDouble(0) < tolerancePerc);
    //sanity check to see if it's within the theoretical standard error of mean
    sampleMeanSEM = sampleMeanDelta.div(normData.theoreticalSEM).max().getDouble(0);
    assertTrue(sampleMeanSEM < 2.6); //99% of the time it should be within this many SEMs

    tolerancePerc = 1; //within 1% - std dev value
    sampleStd = myNormalizer.getStd();
    sampleStdDelta = Transforms.abs(sampleStd.sub(normData.theoreticalStd));

    double actualmaxDiff = sampleStdDelta.div(normData.theoreticalStd).max().mul(100).getDouble(0);
    assertTrue(actualmaxDiff < tolerancePerc);

    tolerancePerc = 1; //within 1%
    normIterator.setPreProcessor(myNormalizer);
    while (normIterator.hasNext()) {
        INDArray before = beforeTransformIterator.next().getFeatures();
        INDArray origBefore = genRandExpectedIter.next().getFeatures();
        INDArray after = normIterator.next().getFeatures();
        INDArray expected = expectedIterator.next().getFeatures();
        delta = Transforms.abs(after.sub(expected));
        deltaPerc = delta.div(Transforms.abs(before.sub(expected)));
        deltaPerc.muli(100);
        maxDeltaPerc = deltaPerc.max(0, 1).getDouble(0);
        /*
        System.out.println("=== BEFORE ===");
        System.out.println(before);
        System.out.println("=== ORIG BEFORE ===");
        System.out.println(origBefore);
        System.out.println("=== AFTER ===");
        System.out.println(after);
        System.out.println("=== SHOULD BE ===");
        System.out.println(expected);
        System.out.println("% diff, "+ maxDeltaPerc);
        */
        assertTrue(maxDeltaPerc < tolerancePerc);
    }
}