Java Code Examples for org.nd4j.linalg.dataset.DataSet#getFeatures()

@Test
public void testMnist() throws Exception {
    ClassPathResource cpr = new ClassPathResource("mnist_first_200.txt");
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(cpr.getTempFileFromArchive()));
    RecordReaderDataSetIterator dsi = new RecordReaderDataSetIterator(rr, 10, 0, 10);

    MnistDataSetIterator iter = new MnistDataSetIterator(10, 200, false, true, false, 0);

    while (dsi.hasNext()) {
        DataSet dsExp = dsi.next();
        DataSet dsAct = iter.next();

        INDArray fExp = dsExp.getFeatures();
        fExp.divi(255);
        INDArray lExp = dsExp.getLabels();

        INDArray fAct = dsAct.getFeatures();
        INDArray lAct = dsAct.getLabels();

        assertEquals(fExp, fAct.castTo(fExp.dataType()));
        assertEquals(lExp, lAct.castTo(lExp.dataType()));
    }
    assertFalse(iter.hasNext());
}
 

/**
 * Converts a set of training instances to a DataSet prepared for the convolution operation using
 * the height, width and number of channels
 *
 * @param height image height
 * @param width image width
 * @param channels number of image channels
 * @param insts the instances to convert
 * @return a DataSet
 */
public static DataSet instancesToConvDataSet(
    Instances insts, int height, int width, int channels) {
  DataSet ds = instancesToDataSet(insts);
  INDArray data = Nd4j.zeros(insts.numInstances(), channels, width, height);
  ds.getFeatures();

  for (int i = 0; i < insts.numInstances(); i++) {
    INDArray row = ds.getFeatures().getRow(i);
    row = row.reshape(1, channels, height, width);
    data.putRow(i, row);
  }

  return new DataSet(data, ds.getLabels());
}
 

@Test
public void testSequenceRecordReaderReset() throws Exception {
    File rootDir = temporaryFolder.newFolder();
    //need to manually extract
    for (int i = 0; i < 3; i++) {
        FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i)));
        FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabels_%d.txt", i)), new File(rootDir, String.format("csvsequencelabels_%d.txt", i)));
    }
    String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt");
    String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt");

    SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ",");
    SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ",");
    featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2));
    labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2));

    SequenceRecordReaderDataSetIterator iter =
                    new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false);

    assertEquals(3, iter.inputColumns());
    assertEquals(4, iter.totalOutcomes());

    int nResets = 5;
    for (int i = 0; i < nResets; i++) {
        iter.reset();
        int count = 0;
        while (iter.hasNext()) {
            DataSet ds = iter.next();
            INDArray features = ds.getFeatures();
            INDArray labels = ds.getLabels();
            assertArrayEquals(new long[] {1, 3, 4}, features.shape());
            assertArrayEquals(new long[] {1, 4, 4}, labels.shape());
            count++;
        }
        assertEquals(3, count);
    }
}
 

/**
 * Converts a set of training instances to a DataSet prepared for the convolution operation using
 * the height, width and number of channels
 *
 * @param height image height
 * @param width image width
 * @param channels number of image channels
 * @param insts the instances to convert
 * @return a DataSet
 */
public static DataSet instancesToConvDataSet(
    Instances insts, int height, int width, int channels) {
  DataSet ds = instancesToDataSet(insts);
  INDArray data = Nd4j.zeros(insts.numInstances(), channels, width, height);
  ds.getFeatures();

  for (int i = 0; i < insts.numInstances(); i++) {
    INDArray row = ds.getFeatures().getRow(i);
    row = row.reshape(1, channels, height, width);
    data.putRow(i, row);
  }

  return new DataSet(data, ds.getLabels());
}
 

@Test
    public void testGradient2dFixedGammaBeta() {
        DataNormalization scaler = new NormalizerMinMaxScaler();
        DataSetIterator iter = new IrisDataSetIterator(150, 150);
        scaler.fit(iter);
        iter.setPreProcessor(scaler);
        DataSet ds = iter.next();
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        for (boolean useLogStd : new boolean[]{true, false}) {
            MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .dataType(DataType.DOUBLE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 1)).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY).build())
                    .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).nOut(3)
                            .build())
                    .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX).nIn(3).nOut(3).build());

            MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
            mln.init();

//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());

            //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
            //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
            //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
            Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev"));
            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input)
                    .labels(labels).excludeParams(excludeParams));

            assertTrue(gradOK);
            TestUtils.testModelSerialization(mln);
        }
    }
 

/**
 * Perform layerwise unsupervised training on a single pre-trainable layer in the network (VAEs, Autoencoders, etc)
 * for the specified number of epochs<br>
 * If the specified layer index (0 to numLayers - 1) is not a pretrainable layer, this is a no-op.
 *
 * @param layerIdx  Index of the layer to train (0 to numLayers-1)
 * @param iter      Training data
 * @param numEpochs Number of epochs to fit the specified layer for
 */
public void pretrainLayer(int layerIdx, DataSetIterator iter, int numEpochs) {
    Preconditions.checkState(numEpochs > 0, "Number of epochs (%s) must be a positive number", numEpochs);

    if (flattenedGradients == null) {
        initGradientsView();
    }
    if (layerIdx >= layers.length) {
        throw new IllegalArgumentException(
                "Cannot pretrain layer: layerIdx (" + layerIdx + ") >= numLayers (" + layers.length + ")");
    }

    Layer layer = layers[layerIdx];
    if (!layer.isPretrainLayer())
        return;

    if(numEpochs > 1 && !iter.resetSupported())
        throw new IllegalStateException("Cannot fit multiple epochs (" + numEpochs + ") on an iterator that doesn't support resetting");

    if (!iter.hasNext() && iter.resetSupported()) {
        iter.reset();
    }

    log.info("Starting unsupervised training on layer " + layerIdx + " for " + numEpochs + " epochs");
    for(int i=0; i<numEpochs; i++ ) {
        if(i > 0)
            iter.reset();

        while (iter.hasNext()) {
            DataSet next = iter.next();
            input = next.getFeatures();
            pretrainLayer(layerIdx, input);
        }
    }

    int ec = getLayer(layerIdx).conf().getEpochCount() + 1;
    getLayer(layerIdx).conf().setEpochCount(ec);
}
 

@Override
public ValidationResult call(String path) throws Exception {
    if (fileSystem == null) {
        Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration();
        try {
            fileSystem = FileSystem.get(new URI(path), c);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    ValidationResult ret = new ValidationResult();
    ret.setCountTotal(1);

    boolean shouldDelete = false;
    boolean loadSuccessful = false;
    DataSet ds = new DataSet();
    Path p = new Path(path);

    if(fileSystem.isDirectory(p)){
        ret.setCountTotal(0);
        return ret;
    }

    if (!fileSystem.exists(p)) {
        ret.setCountMissingFile(1);
        return ret;
    }

    try (FSDataInputStream inputStream = fileSystem.open(p, BUFFER_SIZE)) {
        ds.load(inputStream);
        loadSuccessful = true;
    } catch (RuntimeException t) {
        shouldDelete = deleteInvalid;
        ret.setCountLoadingFailure(1);
    }

    boolean isValid = loadSuccessful;
    if (loadSuccessful) {
        //Validate
        if (ds.getFeatures() == null) {
            ret.setCountMissingFeatures(1);
            isValid = false;
        } else {
            if(featuresShape != null && !validateArrayShape(featuresShape, ds.getFeatures())){
                ret.setCountInvalidFeatures(1);
                isValid = false;
            }
        }

        if(ds.getLabels() == null){
            ret.setCountMissingLabels(1);
            isValid = false;
        } else {
            if(labelsShape != null && !validateArrayShape(labelsShape, ds.getLabels())){
                ret.setCountInvalidLabels(1);
                isValid = false;
            }
        }

        if(!isValid && deleteInvalid){
            shouldDelete = true;
        }
    }

    if (isValid) {
        ret.setCountTotalValid(1);
    } else {
        ret.setCountTotalInvalid(1);
    }

    if (shouldDelete) {
        fileSystem.delete(p, false);
        ret.setCountInvalidDeleted(1);
    }

    return ret;
}
 

@Test
    public void testGradientCNNL1L2MLN() {
        if(this.format != CNN2DFormat.NCHW) //Only test NCHW due to flat input format...
            return;

        //Parameterized test, testing combinations of:
        // (a) activation function
        // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
        // (c) Loss function (with specified output activations)

        DataSet ds = new IrisDataSetIterator(150, 150).next();
        ds.normalizeZeroMeanZeroUnitVariance();
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        //use l2vals[i] with l1vals[i]
        double[] l2vals = {0.4, 0.0, 0.4, 0.4};
        double[] l1vals = {0.0, 0.0, 0.5, 0.0};
        double[] biasL2 = {0.0, 0.0, 0.0, 0.2};
        double[] biasL1 = {0.0, 0.0, 0.6, 0.0};
        Activation[] activFns = {Activation.SIGMOID, Activation.TANH, Activation.ELU, Activation.SOFTPLUS};
        boolean[] characteristic = {false, true, false, true}; //If true: run some backprop steps first

        LossFunctions.LossFunction[] lossFunctions =
                {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE};
        Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.IDENTITY}; //i.e., lossFunctions[i] used with outputActivations[i] here

        for( int i=0; i<l2vals.length; i++ ){
            Activation afn = activFns[i];
            boolean doLearningFirst = characteristic[i];
            LossFunctions.LossFunction lf = lossFunctions[i];
            Activation outputActivation = outputActivations[i];
            double l2 = l2vals[i];
            double l1 = l1vals[i];

            MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                    .dataType(DataType.DOUBLE)
                    .l2(l2).l1(l1).l2Bias(biasL2[i]).l1Bias(biasL1[i])
                    .optimizationAlgo(
                            OptimizationAlgorithm.CONJUGATE_GRADIENT)
                    .seed(12345L).list()
                    .layer(0, new ConvolutionLayer.Builder(new int[]{1, 1}).nIn(1).nOut(6)
                            .weightInit(WeightInit.XAVIER).activation(afn)
                            .updater(new NoOp()).build())
                    .layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3)
                            .weightInit(WeightInit.XAVIER).updater(new NoOp()).build())

                    .setInputType(InputType.convolutionalFlat(1, 4, 1));

            MultiLayerConfiguration conf = builder.build();

            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            String testName = new Object() {
            }.getClass().getEnclosingMethod().getName();

            if (doLearningFirst) {
                //Run a number of iterations of learning
                mln.setInput(ds.getFeatures());
                mln.setLabels(ds.getLabels());
                mln.computeGradientAndScore();
                double scoreBefore = mln.score();
                for (int j = 0; j < 10; j++)
                    mln.fit(ds);
                mln.computeGradientAndScore();
                double scoreAfter = mln.score();
                //Can't test in 'characteristic mode of operation' if not learning
                String msg = testName
                        + "- score did not (sufficiently) decrease during learning - activationFn="
                        + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                        + ", doLearningFirst=" + doLearningFirst + " (before=" + scoreBefore
                        + ", scoreAfter=" + scoreAfter + ")";
                assertTrue(msg, scoreAfter < 0.8 * scoreBefore);
            }

            if (PRINT_RESULTS) {
                System.out.println(testName + "- activationFn=" + afn + ", lossFn=" + lf
                        + ", outputActivation=" + outputActivation + ", doLearningFirst="
                        + doLearningFirst);
//                for (int j = 0; j < mln.getnLayers(); j++)
//                    System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

            assertTrue(gradOK);
            TestUtils.testModelSerialization(mln);
        }
    }
 

@Test
public void testBackwardIrisBasic() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();

    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same outputs
    Nd4j.getRandom().setSeed(12345);
    int nParams = (4 * 5 + 5) + (5 * 3 + 3);
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());

    INDArray input = ds.getFeatures();
    INDArray labels = ds.getLabels();
    graph.setInput(0, input.dup());
    graph.setLabel(0, labels.dup());

    net.setInput(input.dup());
    net.setLabels(labels.dup());

    //Compute gradients
    net.computeGradientAndScore();
    Pair<Gradient, Double> netGradScore = net.gradientAndScore();

    graph.computeGradientAndScore();
    Pair<Gradient, Double> graphGradScore = graph.gradientAndScore();

    assertEquals(netGradScore.getSecond(), graphGradScore.getSecond(), 1e-3);

    //Compare gradients
    Gradient netGrad = netGradScore.getFirst();
    Gradient graphGrad = graphGradScore.getFirst();

    assertNotNull(graphGrad);
    assertEquals(netGrad.gradientForVariable().size(), graphGrad.gradientForVariable().size());

    assertEquals(netGrad.getGradientFor("0_W"), graphGrad.getGradientFor("firstLayer_W"));
    assertEquals(netGrad.getGradientFor("0_b"), graphGrad.getGradientFor("firstLayer_b"));
    assertEquals(netGrad.getGradientFor("1_W"), graphGrad.getGradientFor("outputLayer_W"));
    assertEquals(netGrad.getGradientFor("1_b"), graphGrad.getGradientFor("outputLayer_b"));
}
 

@Test
    public void testBasicIris() {
        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                        .dataType(DataType.DOUBLE)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .dist(new NormalDistribution(0, 1)).updater(new NoOp())
                        .graphBuilder().addInputs("input")
                        .addLayer("firstLayer",
                                        new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(),
                                        "input")
                        .addLayer("outputLayer",
                                        new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                                        .activation(Activation.SOFTMAX).nIn(5).nOut(3).build(),
                                        "firstLayer")
                        .setOutputs("outputLayer").build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        Nd4j.getRandom().setSeed(12345);
        long nParams = graph.numParams();
        INDArray newParams = Nd4j.rand(new long[]{1, nParams});
        graph.setParams(newParams);

        DataSet ds = new IrisDataSetIterator(150, 150).next();
        INDArray min = ds.getFeatures().min(0);
        INDArray max = ds.getFeatures().max(0);
        ds.getFeatures().subiRowVector(min).diviRowVector(max.sub(min));
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        if (PRINT_RESULTS) {
            System.out.println("testBasicIris()");
//            for (int j = 0; j < graph.getNumLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                .labels(new INDArray[]{labels}));

        String msg = "testBasicIris()";
        assertTrue(msg, gradOK);
        TestUtils.testModelSerialization(graph);
    }
 

@Test
    public void testAutoEncoder() {
        //As above (testGradientMLP2LayerIrisSimple()) but with L2, L1, and both L2/L1 applied
        //Need to run gradient through updater, so that L2 can be applied

        Activation[] activFns = {Activation.SIGMOID, Activation.TANH};
        boolean[] characteristic = {false, true}; //If true: run some backprop steps first

        LossFunction[] lossFunctions = {LossFunction.MCXENT, LossFunction.MSE};
        Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH};

        DataNormalization scaler = new NormalizerMinMaxScaler();
        DataSetIterator iter = new IrisDataSetIterator(150, 150);
        scaler.fit(iter);
        iter.setPreProcessor(scaler);
        DataSet ds = iter.next();
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        NormalizerStandardize norm = new NormalizerStandardize();
        norm.fit(ds);
        norm.transform(ds);

        double[] l2vals = {0.2, 0.0, 0.2};
        double[] l1vals = {0.0, 0.3, 0.3}; //i.e., use l2vals[i] with l1vals[i]

        for (Activation afn : activFns) {
            for (boolean doLearningFirst : characteristic) {
                for (int i = 0; i < lossFunctions.length; i++) {
                    for (int k = 0; k < l2vals.length; k++) {
                        LossFunction lf = lossFunctions[i];
                        Activation outputActivation = outputActivations[i];
                        double l2 = l2vals[k];
                        double l1 = l1vals[k];

                        Nd4j.getRandom().setSeed(12345);
                        MultiLayerConfiguration conf =
                                        new NeuralNetConfiguration.Builder()
                                                        .dataType(DataType.DOUBLE)
                                                        .updater(new NoOp())
                                                        .l2(l2).l1(l1)
                                                        .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT)
                                                        .seed(12345L)
                                                        .dist(new NormalDistribution(0, 1))
                                                        .list().layer(0,
                                                                        new AutoEncoder.Builder().nIn(4).nOut(3)
                                                                                        .activation(afn).build())
                                                        .layer(1, new OutputLayer.Builder(lf).nIn(3).nOut(3)
                                                                        .activation(outputActivation).build())
                                                        .build();

                        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                        mln.init();

                        String msg;
                        if (doLearningFirst) {
                            //Run a number of iterations of learning
                            mln.setInput(ds.getFeatures());
                            mln.setLabels(ds.getLabels());
                            mln.computeGradientAndScore();
                            double scoreBefore = mln.score();
                            for (int j = 0; j < 10; j++)
                                mln.fit(ds);
                            mln.computeGradientAndScore();
                            double scoreAfter = mln.score();
                            //Can't test in 'characteristic mode of operation' if not learning
                            msg = "testGradMLP2LayerIrisSimple() - score did not (sufficiently) decrease during learning - activationFn="
                                            + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                                            + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1
                                            + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")";
                            assertTrue(msg, scoreAfter < scoreBefore);
                        }

                        msg = "testGradMLP2LayerIrisSimple() - activationFn=" + afn + ", lossFn=" + lf
                                        + ", outputActivation=" + outputActivation + ", doLearningFirst="
                                        + doLearningFirst + ", l2=" + l2 + ", l1=" + l1;
                        if (PRINT_RESULTS) {
                            System.out.println(msg);
//                            for (int j = 0; j < mln.getnLayers(); j++)
//                                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                        }

                        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
                        assertTrue(msg, gradOK);
                        TestUtils.testModelSerialization(mln);
                    }
                }
            }
        }
    }
 

@Test
    public void testBasicIrisWithElementWiseNodeInputSizeGreaterThanTwo() {

        ElementWiseVertex.Op[] ops =
                        new ElementWiseVertex.Op[] {ElementWiseVertex.Op.Add, ElementWiseVertex.Op.Product, ElementWiseVertex.Op.Average, ElementWiseVertex.Op.Max};

        for (ElementWiseVertex.Op op : ops) {

            Nd4j.getRandom().setSeed(12345);
            ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                            .dataType(DataType.DOUBLE)
                            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                            .dist(new NormalDistribution(0, 1))
                            .updater(new NoOp()).graphBuilder().addInputs("input")
                            .addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(),
                                            "input")
                            .addLayer("l2", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.SIGMOID)
                                            .build(), "input")
                            .addLayer("l3", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(),
                                            "input")
                            .addVertex("elementwise", new ElementWiseVertex(op), "l1", "l2", "l3")
                            .addLayer("outputLayer",
                                            new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                                            .activation(Activation.SOFTMAX).nIn(5).nOut(3).build(),
                                            "elementwise")
                            .setOutputs("outputLayer").build();

            ComputationGraph graph = new ComputationGraph(conf);
            graph.init();

            int numParams = (4 * 5 + 5) + (4 * 5 + 5) + (4 * 5 + 5) + (5 * 3 + 3);
            assertEquals(numParams, graph.numParams());

            Nd4j.getRandom().setSeed(12345);
            long nParams = graph.numParams();
            INDArray newParams = Nd4j.rand(new long[]{1, nParams});
            graph.setParams(newParams);

            DataSet ds = new IrisDataSetIterator(150, 150).next();
            INDArray min = ds.getFeatures().min(0);
            INDArray max = ds.getFeatures().max(0);
            ds.getFeatures().subiRowVector(min).diviRowVector(max.sub(min));
            INDArray input = ds.getFeatures();
            INDArray labels = ds.getLabels();

            if (PRINT_RESULTS) {
                System.out.println("testBasicIrisWithElementWiseVertex(op=" + op + ")");
//                for (int j = 0; j < graph.getNumLayers(); j++)
//                    System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                    .labels(new INDArray[]{labels}));

            String msg = "testBasicIrisWithElementWiseVertex(op=" + op + ")";
            assertTrue(msg, gradOK);
            TestUtils.testModelSerialization(graph);
        }
    }
 

@Test
public void testDataVecSequencePairDataSetFunction() throws Exception {
    JavaSparkContext sc = getContext();

    File f = testDir.newFolder();
    ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/");
    cpr.copyDirectory(f);
    String path = f.getAbsolutePath() + "/*";

    PathToKeyConverter pathConverter = new PathToKeyConverterFilename();
    JavaPairRDD<Text, BytesPairWritable> toWrite =
                    DataVecSparkUtil.combineFilesForSequenceFile(sc, path, path, pathConverter);

    Path p = testDir.newFolder("dl4j_testSeqPairFn").toPath();
    p.toFile().deleteOnExit();
    String outPath = p.toString() + "/out";
    new File(outPath).deleteOnExit();
    toWrite.saveAsNewAPIHadoopFile(outPath, Text.class, BytesPairWritable.class, SequenceFileOutputFormat.class);

    //Load from sequence file:
    JavaPairRDD<Text, BytesPairWritable> fromSeq = sc.sequenceFile(outPath, Text.class, BytesPairWritable.class);

    SequenceRecordReader srr1 = new CSVSequenceRecordReader(1, ",");
    SequenceRecordReader srr2 = new CSVSequenceRecordReader(1, ",");
    PairSequenceRecordReaderBytesFunction psrbf = new PairSequenceRecordReaderBytesFunction(srr1, srr2);
    JavaRDD<Tuple2<List<List<Writable>>, List<List<Writable>>>> writables = fromSeq.map(psrbf);

    //Map to DataSet:
    DataVecSequencePairDataSetFunction pairFn = new DataVecSequencePairDataSetFunction();
    JavaRDD<DataSet> data = writables.map(pairFn);
    List<DataSet> sparkData = data.collect();


    //Now: do the same thing locally (SequenceRecordReaderDataSetIterator) and compare
    String featuresPath = FilenameUtils.concat(f.getAbsolutePath(), "csvsequence_%d.txt");

    SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ",");
    SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ",");
    featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2));
    labelReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2));

    SequenceRecordReaderDataSetIterator iter =
                    new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, -1, true);

    List<DataSet> localData = new ArrayList<>(3);
    while (iter.hasNext())
        localData.add(iter.next());

    assertEquals(3, sparkData.size());
    assertEquals(3, localData.size());

    for (int i = 0; i < 3; i++) {
        //Check shapes etc. data sets order may differ for spark vs. local
        DataSet dsSpark = sparkData.get(i);
        DataSet dsLocal = localData.get(i);

        assertNull(dsSpark.getFeaturesMaskArray());
        assertNull(dsSpark.getLabelsMaskArray());

        INDArray fSpark = dsSpark.getFeatures();
        INDArray fLocal = dsLocal.getFeatures();
        INDArray lSpark = dsSpark.getLabels();
        INDArray lLocal = dsLocal.getLabels();

        val s = new long[] {1, 3, 4}; //1 example, 3 values, 3 time steps
        assertArrayEquals(s, fSpark.shape());
        assertArrayEquals(s, fLocal.shape());
        assertArrayEquals(s, lSpark.shape());
        assertArrayEquals(s, lLocal.shape());
    }


    //Check that results are the same (order not withstanding)
    boolean[] found = new boolean[3];
    for (int i = 0; i < 3; i++) {
        int foundIndex = -1;
        DataSet ds = sparkData.get(i);
        for (int j = 0; j < 3; j++) {
            if (ds.equals(localData.get(j))) {
                if (foundIndex != -1)
                    fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen)
                foundIndex = j;
                if (found[foundIndex])
                    fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list
                found[foundIndex] = true; //mark this one as seen before
            }
        }
    }
    int count = 0;
    for (boolean b : found)
        if (b)
            count++;
    assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions
}
 

@Test
    public void testBasicIrisWithMerging() {
        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                        .dataType(DataType.DOUBLE)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .dist(new NormalDistribution(0, 1)).updater(new NoOp())
                        .graphBuilder().addInputs("input")
                        .addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(),
                                        "input")
                        .addLayer("l2", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(),
                                        "input")
                        .addVertex("merge", new MergeVertex(), "l1", "l2")
                        .addLayer("outputLayer",
                                        new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                                        .activation(Activation.SOFTMAX).nIn(5 + 5).nOut(3).build(),
                                        "merge")
                        .setOutputs("outputLayer").build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        int numParams = (4 * 5 + 5) + (4 * 5 + 5) + (10 * 3 + 3);
        assertEquals(numParams, graph.numParams());

        Nd4j.getRandom().setSeed(12345);
        long nParams = graph.numParams();
        INDArray newParams = Nd4j.rand(new long[]{1, nParams});
        graph.setParams(newParams);

        DataSet ds = new IrisDataSetIterator(150, 150).next();
        INDArray min = ds.getFeatures().min(0);
        INDArray max = ds.getFeatures().max(0);
        ds.getFeatures().subiRowVector(min).diviRowVector(max.sub(min));
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        if (PRINT_RESULTS) {
            System.out.println("testBasicIrisWithMerging()");
//            for (int j = 0; j < graph.getNumLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                .labels(new INDArray[]{labels}));

        String msg = "testBasicIrisWithMerging()";
        assertTrue(msg, gradOK);
        TestUtils.testModelSerialization(graph);
    }
 

@Test
public void when_dataSetIsNHWC_expect_dataSetTransformedToNCHW() {
    // Assemble
    int numChannels = 3;
    int height = 5;
    int width = 4;
    PermuteDataSetPreProcessor sut = new PermuteDataSetPreProcessor(PermuteDataSetPreProcessor.PermutationTypes.NHWCtoNCHW);
    INDArray input = Nd4j.create(1, height, width, numChannels);
    for(int c = 0; c < numChannels; ++c) {
        for(int h = 0; h < height; ++h) {
            for(int w = 0; w < width; ++w) {
                input.putScalar(new int[] { 0, h, w, c }, c*100.0 + h*10.0 + w);
            }
        }
    }
    DataSet ds = new DataSet(input, null);

    // Act
    sut.preProcess(ds);

    // Assert
    INDArray result = ds.getFeatures();
    long[] shape = result.shape();
    assertEquals(1, shape[0]);
    assertEquals(numChannels, shape[1]);
    assertEquals(height, shape[2]);
    assertEquals(width, shape[3]);

    assertEquals(0.0, result.getDouble(0, 0, 0, 0), 0.0);
    assertEquals(1.0, result.getDouble(0, 0, 0, 1), 0.0);
    assertEquals(2.0, result.getDouble(0, 0, 0, 2), 0.0);
    assertEquals(3.0, result.getDouble(0, 0, 0, 3), 0.0);

    assertEquals(110.0, result.getDouble(0, 1, 1, 0), 0.0);
    assertEquals(111.0, result.getDouble(0, 1, 1, 1), 0.0);
    assertEquals(112.0, result.getDouble(0, 1, 1, 2), 0.0);
    assertEquals(113.0, result.getDouble(0, 1, 1, 3), 0.0);

    assertEquals(210.0, result.getDouble(0, 2, 1, 0), 0.0);
    assertEquals(211.0, result.getDouble(0, 2, 1, 1), 0.0);
    assertEquals(212.0, result.getDouble(0, 2, 1, 2), 0.0);
    assertEquals(213.0, result.getDouble(0, 2, 1, 3), 0.0);

}
 

@Test
public void when_dataSetIsNCHW_expect_dataSetTransformedToNHWC() {
    // Assemble
    int numChannels = 3;
    int height = 5;
    int width = 4;
    PermuteDataSetPreProcessor sut = new PermuteDataSetPreProcessor(PermuteDataSetPreProcessor.PermutationTypes.NCHWtoNHWC);
    INDArray input = Nd4j.create(1, numChannels, height, width);
    for(int c = 0; c < numChannels; ++c) {
        for(int h = 0; h < height; ++h) {
            for(int w = 0; w < width; ++w) {
                input.putScalar(0, c, h, w, c*100.0 + h*10.0 + w);
            }
        }
    }
    DataSet ds = new DataSet(input, null);

    // Act
    sut.preProcess(ds);

    // Assert
    INDArray result = ds.getFeatures();
    long[] shape = result.shape();
    assertEquals(1, shape[0]);
    assertEquals(height, shape[1]);
    assertEquals(width, shape[2]);
    assertEquals(numChannels, shape[3]);

    assertEquals(0.0, result.getDouble(0, 0, 0, 0), 0.0);
    assertEquals(1.0, result.getDouble(0, 0, 1, 0), 0.0);
    assertEquals(2.0, result.getDouble(0, 0, 2, 0), 0.0);
    assertEquals(3.0, result.getDouble(0, 0, 3, 0), 0.0);

    assertEquals(110.0, result.getDouble(0, 1, 0, 1), 0.0);
    assertEquals(111.0, result.getDouble(0, 1, 1, 1), 0.0);
    assertEquals(112.0, result.getDouble(0, 1, 2, 1), 0.0);
    assertEquals(113.0, result.getDouble(0, 1, 3, 1), 0.0);

    assertEquals(210.0, result.getDouble(0, 1, 0, 2), 0.0);
    assertEquals(211.0, result.getDouble(0, 1, 1, 2), 0.0);
    assertEquals(212.0, result.getDouble(0, 1, 2, 2), 0.0);
    assertEquals(213.0, result.getDouble(0, 1, 3, 2), 0.0);

}
 

@Test
    public void testBasicIrisWithElementWiseNode() {

        ElementWiseVertex.Op[] ops = new ElementWiseVertex.Op[] {ElementWiseVertex.Op.Add,
                        ElementWiseVertex.Op.Subtract, ElementWiseVertex.Op.Product, ElementWiseVertex.Op.Average, ElementWiseVertex.Op.Max};

        for (ElementWiseVertex.Op op : ops) {

            Nd4j.getRandom().setSeed(12345);
            ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                            .dataType(DataType.DOUBLE)
                            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                            .dist(new NormalDistribution(0, 1))
                            .updater(new NoOp()).graphBuilder().addInputs("input")
                            .addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(),
                                            "input")
                            .addLayer("l2", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.SIGMOID)
                                            .build(), "input")
                            .addVertex("elementwise", new ElementWiseVertex(op), "l1", "l2")
                            .addLayer("outputLayer",
                                            new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                                            .activation(Activation.SOFTMAX).nIn(5).nOut(3).build(),
                                            "elementwise")
                            .setOutputs("outputLayer").build();

            ComputationGraph graph = new ComputationGraph(conf);
            graph.init();

            int numParams = (4 * 5 + 5) + (4 * 5 + 5) + (5 * 3 + 3);
            assertEquals(numParams, graph.numParams());

            Nd4j.getRandom().setSeed(12345);
            long nParams = graph.numParams();
            INDArray newParams = Nd4j.rand(new long[]{1, nParams});
            graph.setParams(newParams);

            DataSet ds = new IrisDataSetIterator(150, 150).next();
            INDArray min = ds.getFeatures().min(0);
            INDArray max = ds.getFeatures().max(0);
            ds.getFeatures().subiRowVector(min).diviRowVector(max.sub(min));
            INDArray input = ds.getFeatures();
            INDArray labels = ds.getLabels();

            if (PRINT_RESULTS) {
                System.out.println("testBasicIrisWithElementWiseVertex(op=" + op + ")");
//                for (int j = 0; j < graph.getNumLayers(); j++)
//                    System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                    .labels(new INDArray[]{labels}));

            String msg = "testBasicIrisWithElementWiseVertex(op=" + op + ")";
            assertTrue(msg, gradOK);
            TestUtils.testModelSerialization(graph);
        }
    }
 

@Test
public void testPreprocessedDataCompGraphMultiDataSet() throws IOException {
    //Test _loading_ of preprocessed MultiDataSet data
    int dataSetObjSize = 5;
    int batchSizePerExecutor = 10;

    String path = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_testpreprocdata3");
    File f = new File(path);
    if (f.exists())
        f.delete();
    f.mkdir();

    DataSetIterator iter = new IrisDataSetIterator(5, 150);
    int i = 0;
    while (iter.hasNext()) {
        File f2 = new File(FilenameUtils.concat(path, "data" + (i++) + ".bin"));
        DataSet ds = iter.next();
        MultiDataSet mds = new MultiDataSet(ds.getFeatures(), ds.getLabels());
        mds.save(f2);
    }

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .graphBuilder().addInputs("in")
                    .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(3)
                                    .activation(Activation.TANH).build(), "in")
                    .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3).activation(Activation.SOFTMAX)
                                                    .build(),
                                    "0")
                    .setOutputs("1").build();

    SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf,
                    new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize)
                                    .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1)
                                    .repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);

    sparkNet.fitMultiDataSet("file:///" + path.replaceAll("\\\\", "/"));

    SparkTrainingStats sts = sparkNet.getSparkTrainingStats();
    int expNumFits = 12; //4 'fits' per averaging (4 executors, 1 averaging freq); 10 examples each -> 40 examples per fit. 150/40 = 3 averagings (round down); 3*4 = 12

    //Unfortunately: perfect partitioning isn't guaranteed by SparkUtils.balancedRandomSplit (esp. if original partitions are all size 1
    // which appears to be occurring at least some of the time), but we should get close to what we expect...
    assertTrue(Math.abs(expNumFits - sts.getValue("ParameterAveragingWorkerFitTimesMs").size()) < 3);

    assertEquals(3, sts.getValue("ParameterAveragingMasterMapPartitionsTimesMs").size());
}
 

@Test
public void testSplittingCSV() throws Exception {
    //Here's the idea: take Iris, and split it up into 2 inputs and 2 output arrays
    //Inputs: columns 0 and 1-2
    //Outputs: columns 3, and 4->OneHot
    //need to manually extract
    RecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(Resources.asFile("iris.txt")));
    RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 10, 4, 3);

    RecordReader rr2 = new CSVRecordReader(0, ',');
    rr2.initialize(new FileSplit(Resources.asFile("iris.txt")));

    MultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2)
                    .addInput("reader", 0, 0).addInput("reader", 1, 2).addOutput("reader", 3, 3)
                    .addOutputOneHot("reader", 4, 3).build();

    while (rrdsi.hasNext()) {
        DataSet ds = rrdsi.next();
        INDArray fds = ds.getFeatures();
        INDArray lds = ds.getLabels();

        MultiDataSet mds = rrmdsi.next();
        assertEquals(2, mds.getFeatures().length);
        assertEquals(2, mds.getLabels().length);
        assertNull(mds.getFeaturesMaskArrays());
        assertNull(mds.getLabelsMaskArrays());
        INDArray[] fmds = mds.getFeatures();
        INDArray[] lmds = mds.getLabels();

        assertNotNull(fmds);
        assertNotNull(lmds);
        for (int i = 0; i < fmds.length; i++)
            assertNotNull(fmds[i]);
        for (int i = 0; i < lmds.length; i++)
            assertNotNull(lmds[i]);

        //Get the subsets of the original iris data
        INDArray expIn1 = fds.get(all(), interval(0,0,true));
        INDArray expIn2 = fds.get(all(), interval(1, 2, true));
        INDArray expOut1 = fds.get(all(), interval(3,3,true));
        INDArray expOut2 = lds;

        assertEquals(expIn1, fmds[0]);
        assertEquals(expIn2, fmds[1]);
        assertEquals(expOut1, lmds[0]);
        assertEquals(expOut2, lmds[1]);
    }
    assertFalse(rrmdsi.hasNext());
}
 

private static void testIrisMiniBatchGradients(int miniBatchSize, int[] hiddenLayerSizes,
                Activation activationFunction) {
    int totalExamples = 10 * miniBatchSize;
    if (totalExamples > 150) {
        totalExamples = miniBatchSize * (150 / miniBatchSize);
    }
    if (miniBatchSize > 150) {
        fail();
    }
    DataSetIterator iris = new IrisDataSetIterator(miniBatchSize, totalExamples);

    MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(hiddenLayerSizes, Activation.SIGMOID));
    network.init();

    Layer[] layers = network.getLayers();
    int nLayers = layers.length;

    while (iris.hasNext()) {
        DataSet data = iris.next();
        INDArray x = data.getFeatures();
        INDArray y = data.getLabels();

        //Do forward pass:
        INDArray[] layerWeights = new INDArray[nLayers];
        INDArray[] layerBiases = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            layerWeights[i] = layers[i].getParam(DefaultParamInitializer.WEIGHT_KEY).dup();
            layerBiases[i] = layers[i].getParam(DefaultParamInitializer.BIAS_KEY).dup();
        }

        INDArray[] layerZs = new INDArray[nLayers];
        INDArray[] layerActivations = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            INDArray layerInput = (i == 0 ? x : layerActivations[i - 1]);
            layerZs[i] = layerInput.castTo(layerWeights[i].dataType()).mmul(layerWeights[i]).addiRowVector(layerBiases[i]);
            layerActivations[i] = (i == nLayers - 1 ? doSoftmax(layerZs[i].dup()) : doSigmoid(layerZs[i].dup()));
        }

        //Do backward pass:
        INDArray[] deltas = new INDArray[nLayers];
        deltas[nLayers - 1] = layerActivations[nLayers - 1].sub(y.castTo(layerActivations[nLayers-1].dataType())); //Out - labels; shape=[miniBatchSize,nOut];
        assertArrayEquals(deltas[nLayers - 1].shape(), new long[] {miniBatchSize, 3});
        for (int i = nLayers - 2; i >= 0; i--) {
            INDArray sigmaPrimeOfZ;
            sigmaPrimeOfZ = doSigmoidDerivative(layerZs[i]);
            INDArray epsilon = layerWeights[i + 1].mmul(deltas[i + 1].transpose()).transpose();
            deltas[i] = epsilon.mul(sigmaPrimeOfZ);
            assertArrayEquals(deltas[i].shape(), new long[] {miniBatchSize, hiddenLayerSizes[i]});
        }

        INDArray[] dLdw = new INDArray[nLayers];
        INDArray[] dLdb = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            INDArray prevActivations = (i == 0 ? x : layerActivations[i - 1]);
            //Raw gradients, so not yet divided by mini-batch size (division is done in BaseUpdater)
            dLdw[i] = deltas[i].transpose().castTo(prevActivations.dataType()).mmul(prevActivations).transpose(); //Shape: [nIn, nOut]
            dLdb[i] = deltas[i].sum(true, 0); //Shape: [1,nOut]

            int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]);
            int nOut = (i < nLayers - 1 ? hiddenLayerSizes[i] : 3);
            assertArrayEquals(dLdw[i].shape(), new long[] {nIn, nOut});
            assertArrayEquals(dLdb[i].shape(), new long[] {1, nOut});
        }


        //Calculate and get gradient, compare to expected
        network.setInput(x);
        network.setLabels(y);
        network.computeGradientAndScore();
        Gradient gradient = network.gradientAndScore().getFirst();

        float eps = 1e-4f;
        for (int i = 0; i < hiddenLayerSizes.length; i++) {
            String wKey = i + "_" + DefaultParamInitializer.WEIGHT_KEY;
            String bKey = i + "_" + DefaultParamInitializer.BIAS_KEY;
            INDArray wGrad = gradient.getGradientFor(wKey);
            INDArray bGrad = gradient.getGradientFor(bKey);
            float[] wGradf = asFloat(wGrad);
            float[] bGradf = asFloat(bGrad);
            float[] expWGradf = asFloat(dLdw[i]);
            float[] expBGradf = asFloat(dLdb[i]);
            assertArrayEquals(wGradf, expWGradf, eps);
            assertArrayEquals(bGradf, expBGradf, eps);
        }
    }
}