Java Code Examples for org.nd4j.linalg.dataset.api.iterator.DataSetIterator#hasNext()

@Test
public void testCompGraphUnderscores() {
    //Problem: underscores in names could be problematic for ComputationGraphUpdater, HistogramIterationListener

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder()
            .addInputs("input")
            .addLayer("first_layer", new DenseLayer.Builder().nIn(4).nOut(5).build(), "input")
            .addLayer("output_layer", new OutputLayer.Builder().nIn(5).nOut(3).activation(Activation.SOFTMAX).build(), "first_layer")
            .setOutputs("output_layer").build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(10, 150);
    while (iris.hasNext()) {
        net.fit(iris.next());
    }
}
 

@Test
public void testRRMDSI5D() {
    int batchSize = 5;

    CustomRecordReader recordReader = new CustomRecordReader();
    DataSetIterator dataIter = new RecordReaderDataSetIterator(recordReader, batchSize,
            1, /* Index of label in records */
            2 /* number of different labels */);

    int count = 0;
    while(dataIter.hasNext()){
        DataSet ds = dataIter.next();

        int offset = 5*count;
        for( int i=0; i<5; i++ ){
            INDArray act = ds.getFeatures().get(interval(i,i,true), all(), all(), all(), all());
            INDArray exp = Nd4j.valueArrayOf(new int[]{1, 1, nZ, nX, nY}, i + offset );
            assertEquals(exp, act);
        }
        count++;
    }

    assertEquals(2, count);
}
 

@Test
public void testDSI(){
    DataSetIterator iter = new RandomDataSetIterator(5, new long[]{3,4}, new long[]{3,5}, RandomDataSetIterator.Values.RANDOM_UNIFORM,
            RandomDataSetIterator.Values.ONE_HOT);

    int count = 0;
    while(iter.hasNext()){
        count++;
        DataSet ds = iter.next();

        assertArrayEquals(new long[]{3,4}, ds.getFeatures().shape());
        assertArrayEquals(new long[]{3,5}, ds.getLabels().shape());

        assertTrue(ds.getFeatures().minNumber().doubleValue() >= 0.0 && ds.getFeatures().maxNumber().doubleValue() <= 1.0);
        assertEquals(Nd4j.ones(3), ds.getLabels().sum(1));
    }
    assertEquals(5, count);
}
 

@Test
public void testMultiOutputEvalSimple(){
    Nd4j.getRandom().setSeed(12345);

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .graphBuilder()
            .addInputs("in")
            .addLayer("out1", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in")
            .addLayer("out2", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in")
            .setOutputs("out1", "out2")
            .build();

    ComputationGraph cg = new ComputationGraph(conf);
    cg.init();

    List<MultiDataSet> list = new ArrayList<>();
    DataSetIterator iter = new IrisDataSetIterator(30, 150);
    while(iter.hasNext()){
        DataSet ds = iter.next();
        list.add(new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{ds.getFeatures()}, new INDArray[]{ds.getLabels(), ds.getLabels()}));
    }

    org.nd4j.evaluation.classification.Evaluation e = new org.nd4j.evaluation.classification.Evaluation();
    org.nd4j.evaluation.regression.RegressionEvaluation e2 = new org.nd4j.evaluation.regression.RegressionEvaluation();
    Map<Integer,org.nd4j.evaluation.IEvaluation[]> evals = new HashMap<>();
    evals.put(0, new org.nd4j.evaluation.IEvaluation[]{e});
    evals.put(1, new org.nd4j.evaluation.IEvaluation[]{e2});

    cg.evaluate(new IteratorMultiDataSetIterator(list.iterator(), 30), evals);

    assertEquals(150, e.getNumRowCounter());
    assertEquals(150, e2.getExampleCountPerColumn().getInt(0));
}
 

/**
 * Perform layerwise unsupervised training on a single pre-trainable layer in the network (VAEs, Autoencoders, etc)
 * for the specified number of epochs<br>
 * If the specified layer index (0 to numLayers - 1) is not a pretrainable layer, this is a no-op.
 *
 * @param layerIdx  Index of the layer to train (0 to numLayers-1)
 * @param iter      Training data
 * @param numEpochs Number of epochs to fit the specified layer for
 */
public void pretrainLayer(int layerIdx, DataSetIterator iter, int numEpochs) {
    Preconditions.checkState(numEpochs > 0, "Number of epochs (%s) must be a positive number", numEpochs);

    if (flattenedGradients == null) {
        initGradientsView();
    }
    if (layerIdx >= layers.length) {
        throw new IllegalArgumentException(
                "Cannot pretrain layer: layerIdx (" + layerIdx + ") >= numLayers (" + layers.length + ")");
    }

    Layer layer = layers[layerIdx];
    if (!layer.isPretrainLayer())
        return;

    if(numEpochs > 1 && !iter.resetSupported())
        throw new IllegalStateException("Cannot fit multiple epochs (" + numEpochs + ") on an iterator that doesn't support resetting");

    if (!iter.hasNext() && iter.resetSupported()) {
        iter.reset();
    }

    log.info("Starting unsupervised training on layer " + layerIdx + " for " + numEpochs + " epochs");
    for(int i=0; i<numEpochs; i++ ) {
        if(i > 0)
            iter.reset();

        while (iter.hasNext()) {
            DataSet next = iter.next();
            input = next.getFeatures();
            pretrainLayer(layerIdx, input);
        }
    }

    int ec = getLayer(layerIdx).conf().getEpochCount() + 1;
    getLayer(layerIdx).conf().setEpochCount(ec);
}
 

@Test
    public void testMLP() {
        //Simple mini-batch test with multiple hidden layers
        MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] {5, 4, 3}, Activation.SIGMOID);
//        System.out.println(conf);
        MultiLayerNetwork network = new MultiLayerNetwork(conf);
        network.init();
        DataSetIterator iter = new IrisDataSetIterator(10, 100);

        while (iter.hasNext()) {
            network.fit(iter.next());
        }
    }
 

/**
 * Counts the number of iterations
 *
 * @param data Instances to iterate
 * @param iter iterator to be tested
 * @param seed Seed
 * @param batchsize Size of the batch which is returned in {@see DataSetIterator#next}
 * @return Number of iterations
 */
private int countIterations(
    Instances data, AbstractInstanceIterator iter, int seed, int batchsize) throws Exception {
  DataSetIterator it = iter.getDataSetIterator(data, seed, batchsize);
  int count = 0;
  while (it.hasNext()) {
    count++;
    Utils.getNext(it);
  }
  return count;
}
 

/**
 * Counts the number of iterations
 *
 * @param data Instances to iterate
 * @param iter iterator to be tested
 * @param seed Seed
 * @param batchsize Size of the batch which is returned in {@see DataSetIterator#next}
 * @return Number of iterations
 */
private int countIterations(
    Instances data, AbstractInstanceIterator iter, int seed, int batchsize) throws Exception {
  DataSetIterator it = iter.getDataSetIterator(data, seed, batchsize);
  int count = 0;
  while (it.hasNext()) {
    count++;
    Utils.getNext(it);
  }
  return count;
}
 

private static void testIrisMiniBatchGradients(int miniBatchSize, int[] hiddenLayerSizes,
                Activation activationFunction) {
    int totalExamples = 10 * miniBatchSize;
    if (totalExamples > 150) {
        totalExamples = miniBatchSize * (150 / miniBatchSize);
    }
    if (miniBatchSize > 150) {
        fail();
    }
    DataSetIterator iris = new IrisDataSetIterator(miniBatchSize, totalExamples);

    MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(hiddenLayerSizes, Activation.SIGMOID));
    network.init();

    Layer[] layers = network.getLayers();
    int nLayers = layers.length;

    while (iris.hasNext()) {
        DataSet data = iris.next();
        INDArray x = data.getFeatures();
        INDArray y = data.getLabels();

        //Do forward pass:
        INDArray[] layerWeights = new INDArray[nLayers];
        INDArray[] layerBiases = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            layerWeights[i] = layers[i].getParam(DefaultParamInitializer.WEIGHT_KEY).dup();
            layerBiases[i] = layers[i].getParam(DefaultParamInitializer.BIAS_KEY).dup();
        }

        INDArray[] layerZs = new INDArray[nLayers];
        INDArray[] layerActivations = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            INDArray layerInput = (i == 0 ? x : layerActivations[i - 1]);
            layerZs[i] = layerInput.castTo(layerWeights[i].dataType()).mmul(layerWeights[i]).addiRowVector(layerBiases[i]);
            layerActivations[i] = (i == nLayers - 1 ? doSoftmax(layerZs[i].dup()) : doSigmoid(layerZs[i].dup()));
        }

        //Do backward pass:
        INDArray[] deltas = new INDArray[nLayers];
        deltas[nLayers - 1] = layerActivations[nLayers - 1].sub(y.castTo(layerActivations[nLayers-1].dataType())); //Out - labels; shape=[miniBatchSize,nOut];
        assertArrayEquals(deltas[nLayers - 1].shape(), new long[] {miniBatchSize, 3});
        for (int i = nLayers - 2; i >= 0; i--) {
            INDArray sigmaPrimeOfZ;
            sigmaPrimeOfZ = doSigmoidDerivative(layerZs[i]);
            INDArray epsilon = layerWeights[i + 1].mmul(deltas[i + 1].transpose()).transpose();
            deltas[i] = epsilon.mul(sigmaPrimeOfZ);
            assertArrayEquals(deltas[i].shape(), new long[] {miniBatchSize, hiddenLayerSizes[i]});
        }

        INDArray[] dLdw = new INDArray[nLayers];
        INDArray[] dLdb = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            INDArray prevActivations = (i == 0 ? x : layerActivations[i - 1]);
            //Raw gradients, so not yet divided by mini-batch size (division is done in BaseUpdater)
            dLdw[i] = deltas[i].transpose().castTo(prevActivations.dataType()).mmul(prevActivations).transpose(); //Shape: [nIn, nOut]
            dLdb[i] = deltas[i].sum(true, 0); //Shape: [1,nOut]

            int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]);
            int nOut = (i < nLayers - 1 ? hiddenLayerSizes[i] : 3);
            assertArrayEquals(dLdw[i].shape(), new long[] {nIn, nOut});
            assertArrayEquals(dLdb[i].shape(), new long[] {1, nOut});
        }


        //Calculate and get gradient, compare to expected
        network.setInput(x);
        network.setLabels(y);
        network.computeGradientAndScore();
        Gradient gradient = network.gradientAndScore().getFirst();

        float eps = 1e-4f;
        for (int i = 0; i < hiddenLayerSizes.length; i++) {
            String wKey = i + "_" + DefaultParamInitializer.WEIGHT_KEY;
            String bKey = i + "_" + DefaultParamInitializer.BIAS_KEY;
            INDArray wGrad = gradient.getGradientFor(wKey);
            INDArray bGrad = gradient.getGradientFor(bKey);
            float[] wGradf = asFloat(wGrad);
            float[] bGradf = asFloat(bGrad);
            float[] expWGradf = asFloat(dLdw[i]);
            float[] expBGradf = asFloat(dLdb[i]);
            assertArrayEquals(wGradf, expWGradf, eps);
            assertArrayEquals(bGradf, expBGradf, eps);
        }
    }
}
 

@Test
public void testPreprocessedDataCompGraphMultiDataSet() throws IOException {
    //Test _loading_ of preprocessed MultiDataSet data
    int dataSetObjSize = 5;
    int batchSizePerExecutor = 10;

    String path = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_testpreprocdata3");
    File f = new File(path);
    if (f.exists())
        f.delete();
    f.mkdir();

    DataSetIterator iter = new IrisDataSetIterator(5, 150);
    int i = 0;
    while (iter.hasNext()) {
        File f2 = new File(FilenameUtils.concat(path, "data" + (i++) + ".bin"));
        DataSet ds = iter.next();
        MultiDataSet mds = new MultiDataSet(ds.getFeatures(), ds.getLabels());
        mds.save(f2);
    }

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .graphBuilder().addInputs("in")
                    .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(3)
                                    .activation(Activation.TANH).build(), "in")
                    .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3).activation(Activation.SOFTMAX)
                                                    .build(),
                                    "0")
                    .setOutputs("1").build();

    SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf,
                    new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize)
                                    .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1)
                                    .repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);

    sparkNet.fitMultiDataSet("file:///" + path.replaceAll("\\\\", "/"));

    SparkTrainingStats sts = sparkNet.getSparkTrainingStats();
    int expNumFits = 12; //4 'fits' per averaging (4 executors, 1 averaging freq); 10 examples each -> 40 examples per fit. 150/40 = 3 averagings (round down); 3*4 = 12

    //Unfortunately: perfect partitioning isn't guaranteed by SparkUtils.balancedRandomSplit (esp. if original partitions are all size 1
    // which appears to be occurring at least some of the time), but we should get close to what we expect...
    assertTrue(Math.abs(expNumFits - sts.getValue("ParameterAveragingWorkerFitTimesMs").size()) < 3);

    assertEquals(3, sts.getValue("ParameterAveragingMasterMapPartitionsTimesMs").size());
}
 

/**
 * The method to use when making predictions for test instances.
 *
 * @param insts the instances to get predictions for
 * @return the class probability estimates (if the class is nominal) or the numeric predictions
 * (if it is numeric)
 * @throws Exception if something goes wrong at prediction time
 */
@Override
public double[][] distributionsForInstances(Instances insts) throws Exception {

  // Do we only have a ZeroR model?
  if (zeroR != null) {
    return zeroR.distributionsForInstances(insts);
  }

  // Process input data to have the same filters applied as the training data
  insts = applyFilters(insts);

  // Get predictions
  final DataSetIterator it = getDataSetIterator(insts, CacheMode.NONE);
  double[][] preds = new double[insts.numInstances()][insts.numClasses()];

  int offset = 0;
  boolean next = it.hasNext();

  // Get predictions batch-wise
  while (next) {
    INDArray predBatch = model.outputSingle(Utils.getNext(it).getFeatures());

    if (arithmeticUnderflow(predBatch))
      throw new DL4JException("NaNs in model output, likely caused by arithmetic underflow");

    int currentBatchSize = (int) predBatch.shape()[0];

    // Build weka distribution output
    for (int i = 0; i < currentBatchSize; i++) {
      for (int j = 0; j < insts.numClasses(); j++) {
        int jResorted = fixLabelIndexIfNominal(j, insts);
        preds[i + offset][j] = predBatch.getDouble(i, jResorted);
      }
    }
    offset += currentBatchSize; // add batchsize as offset
    boolean hasInstancesLeft = offset < insts.numInstances();
    next = it.hasNext() || hasInstancesLeft;
  }

  // Fix classes
  for (int i = 0; i < preds.length; i++) {
    // only normalise if we're dealing with classification
    if (preds[i].length > 1) {
      weka.core.Utils.normalize(preds[i]);
    } else {
      // Rescale numeric classes with the computed coefficients in the
      // initialization phase
      preds[i][0] = preds[i][0] * x1 + x0;
    }
  }
  return preds;
}
 

@Test
public void testBruteForce() {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
    //X_scaled = X_std * (max - min) + min
    // Dataset features are scaled consecutive natural numbers
    int nSamples = 500;
    int x = 4, y = 2, z = 3;

    INDArray featureX = Nd4j.linspace(1, nSamples, nSamples).reshape(nSamples, 1);
    INDArray featureY = featureX.mul(y);
    INDArray featureZ = featureX.mul(z);
    featureX.muli(x);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);

    //expected min and max
    INDArray theoreticalMin = Nd4j.create(new double[] {x, y, z});
    INDArray theoreticalMax = Nd4j.create(new double[] {nSamples * x, nSamples * y, nSamples * z});
    INDArray theoreticalRange = theoreticalMax.sub(theoreticalMin);

    NormalizerMinMaxScaler myNormalizer = new NormalizerMinMaxScaler();
    myNormalizer.fit(sampleDataSet);

    INDArray minDataSet = myNormalizer.getMin();
    INDArray maxDataSet = myNormalizer.getMax();
    INDArray minDiff = minDataSet.sub(theoreticalMin).max(1);
    INDArray maxDiff = maxDataSet.sub(theoreticalMax).max(1);
    assertEquals(minDiff.getDouble(0, 0), 0.0, 0.000000001);
    assertEquals(maxDiff.max(1).getDouble(0, 0), 0.0, 0.000000001);

    // SAME TEST WITH THE ITERATOR
    int bSize = 1;
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);
    myNormalizer.fit(sampleIter);
    minDataSet = myNormalizer.getMin();
    maxDataSet = myNormalizer.getMax();
    assertEquals(minDataSet.sub(theoreticalMin).max(1).getDouble(0, 0), 0.0, 0.000000001);
    assertEquals(maxDataSet.sub(theoreticalMax).max(1).getDouble(0, 0), 0.0, 0.000000001);

    sampleIter.setPreProcessor(myNormalizer);
    INDArray actual, expected, delta;
    int i = 1;
    while (sampleIter.hasNext()) {
        expected = theoreticalMin.mul(i - 1).div(theoreticalRange);
        actual = sampleIter.next().getFeatures();
        delta = Transforms.abs(actual.sub(expected));
        assertTrue(delta.max(1).getDouble(0, 0) < 0.0001);
        i++;
    }

}
 

@Test
    public void testFitViaStringPaths() throws Exception {

        Path tempDir = testDir.newFolder("DL4J-testFitViaStringPaths").toPath();
        File tempDirF = tempDir.toFile();
        tempDirF.deleteOnExit();

        int dataSetObjSize = 5;
        int batchSizePerExecutor = 25;
        DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 1000, false);
        int i = 0;
        while (iter.hasNext()) {
            File nextFile = new File(tempDirF, i + ".bin");
            DataSet ds = iter.next();
            ds.save(nextFile);
            i++;
        }

        System.out.println("Saved to: " + tempDirF.getAbsolutePath());



        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp())
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                        .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50)
                                        .activation(Activation.TANH).build())
                        .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                        LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10)
                                                        .activation(Activation.SOFTMAX).build())
                        .build();

        SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf,
                        new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize)
                                        .workerPrefetchNumBatches(5).batchSizePerWorker(batchSizePerExecutor)
                                        .averagingFrequency(1).repartionData(Repartition.Always).build());
        sparkNet.setCollectTrainingStats(true);


        //List files:
        Configuration config = new Configuration();
        FileSystem hdfs = FileSystem.get(tempDir.toUri(), config);
        RemoteIterator<LocatedFileStatus> fileIter =
                        hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false);

        List<String> paths = new ArrayList<>();
        while (fileIter.hasNext()) {
            String path = fileIter.next().getPath().toString();
            paths.add(path);
        }

        INDArray paramsBefore = sparkNet.getNetwork().params().dup();
        JavaRDD<String> pathRdd = sc.parallelize(paths);
        sparkNet.fitPaths(pathRdd);

        INDArray paramsAfter = sparkNet.getNetwork().params().dup();
        assertNotEquals(paramsBefore, paramsAfter);

        SparkTrainingStats stats = sparkNet.getSparkTrainingStats();
//        System.out.println(stats.statsAsString());
        stats.statsAsString();

        sparkNet.getTrainingMaster().deleteTempFiles(sc);
    }
 

@Test
public void testEvaluationAndRocMDS() {
    for( int evalWorkers : new int[]{1, 4, 8}) {

        DataSetIterator iter = new IrisDataSetIterator(5, 150);

        //Make a 2-class version of iris:
        List<MultiDataSet> l = new ArrayList<>();
        iter.reset();
        while (iter.hasNext()) {
            DataSet ds = iter.next();
            INDArray newL = Nd4j.create(ds.getLabels().size(0), 2);
            newL.putColumn(0, ds.getLabels().getColumn(0));
            newL.putColumn(1, ds.getLabels().getColumn(1));
            newL.getColumn(1).addi(ds.getLabels().getColumn(2));

            MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(ds.getFeatures(), newL);
            l.add(mds);
        }

        MultiDataSetIterator mdsIter = new IteratorMultiDataSetIterator(l.iterator(), 5);

        ComputationGraph cg = getBasicNetIris2Class();

        IEvaluation[] es = cg.doEvaluation(mdsIter, new Evaluation(), new ROC(32));
        Evaluation e = (Evaluation) es[0];
        ROC roc = (ROC) es[1];


        SparkComputationGraph scg = new SparkComputationGraph(sc, cg, null);
        scg.setDefaultEvaluationWorkers(evalWorkers);

        JavaRDD<MultiDataSet> rdd = sc.parallelize(l);
        rdd = rdd.repartition(20);

        IEvaluation[] es2 = scg.doEvaluationMDS(rdd, 5, new Evaluation(), new ROC(32));
        Evaluation e2 = (Evaluation) es2[0];
        ROC roc2 = (ROC) es2[1];


        assertEquals(e2.accuracy(), e.accuracy(), 1e-3);
        assertEquals(e2.f1(), e.f1(), 1e-3);
        assertEquals(e2.getNumRowCounter(), e.getNumRowCounter(), 1e-3);
        assertEquals(e2.falseNegatives(), e.falseNegatives());
        assertEquals(e2.falsePositives(), e.falsePositives());
        assertEquals(e2.trueNegatives(), e.trueNegatives());
        assertEquals(e2.truePositives(), e.truePositives());
        assertEquals(e2.precision(), e.precision(), 1e-3);
        assertEquals(e2.recall(), e.recall(), 1e-3);
        assertEquals(e2.getConfusionMatrix(), e.getConfusionMatrix());

        assertEquals(roc.calculateAUC(), roc2.calculateAUC(), 1e-5);
        assertEquals(roc.calculateAUCPR(), roc2.calculateAUCPR(), 1e-5);
    }
}
 

public static void main(String[] args) throws Exception {
    //number of rows and columns in the input pictures
    final int numRows = 28;
    final int numColumns = 28;
    int outputNum = 10; // number of output classes
    int batchSize = 128; // batch size for each epoch
    int rngSeed = 123; // random number seed for reproducibility
    int numEpochs = 15; // number of epochs to perform

    //Get the DataSetIterators:
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, rngSeed);
    DataSetIterator mnistTest = new MnistDataSetIterator(batchSize, false, rngSeed);


    log.info("Build model....");
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(rngSeed) //include a random seed for reproducibility
            // use stochastic gradient descent as an optimization algorithm
            .updater(new Nesterovs(0.006, 0.9))
            .l2(1e-4)
            .list()
            .layer(0, new DenseLayer.Builder() //create the first, input layer with xavier initialization
                    // batchSize, features
                    .nIn(numRows * numColumns)
                    .nOut(1000)
                    .activation(Activation.RELU)
                    .weightInit(WeightInit.XAVIER)
                    .build())
            .layer(1, new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD) //create hidden layer
                    .nIn(1000)
                    .nOut(outputNum)
                    .activation(Activation.SOFTMAX)
                    .weightInit(WeightInit.XAVIER)
                    .build())
            .pretrain(false).backprop(true) //use backpropagation to adjust weights
            .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    //print the score with every 1 iteration
    model.setListeners(new ScoreIterationListener(1));

    log.info("Train model....");
    for( int i=0; i<numEpochs; i++ ){
        model.fit(mnistTrain);
    }


    log.info("Evaluate model....");
    Evaluation eval = new Evaluation(outputNum); //create an evaluation object with 10 possible classes
    while(mnistTest.hasNext()){
        DataSet next = mnistTest.next();
        INDArray output = model.output(next.getFeatures(), false); //get the networks prediction
        eval.eval(next.getLabels(), output); //check the prediction against the true class
    }

    try {
        ModelSerializer.writeModel(model, new File("model/SingleLayerModel.zip"), false);
    } catch (IOException e) {
        e.printStackTrace();
    }

    log.info(eval.stats());
    log.info("****************Example finished********************");

}
 

@Test
public void testBruteForce() {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
    //X_scaled = X_std * (max - min) + min
    // Dataset features are scaled consecutive natural numbers
    int nSamples = 500;
    int x = 4, y = 2, z = 3;

    INDArray featureX = Nd4j.linspace(1, nSamples, nSamples).reshape(nSamples, 1);
    INDArray featureY = featureX.mul(y);
    INDArray featureZ = featureX.mul(z);
    featureX.muli(x);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);

    //expected min and max
    INDArray theoreticalMin = Nd4j.create(new double[] {x, y, z}, new long[]{1,3});
    INDArray theoreticalMax = Nd4j.create(new double[] {nSamples * x, nSamples * y, nSamples * z}, new long[]{1,3});
    INDArray theoreticalRange = theoreticalMax.sub(theoreticalMin);

    NormalizerMinMaxScaler myNormalizer = new NormalizerMinMaxScaler();
    myNormalizer.fit(sampleDataSet);

    INDArray minDataSet = myNormalizer.getMin();
    INDArray maxDataSet = myNormalizer.getMax();
    INDArray minDiff = minDataSet.sub(theoreticalMin).max();
    INDArray maxDiff = maxDataSet.sub(theoreticalMax).max();
    assertEquals(minDiff.getDouble(0), 0.0, 0.000000001);
    assertEquals(maxDiff.max().getDouble(0), 0.0, 0.000000001);

    // SAME TEST WITH THE ITERATOR
    int bSize = 1;
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);
    myNormalizer.fit(sampleIter);
    minDataSet = myNormalizer.getMin();
    maxDataSet = myNormalizer.getMax();
    assertEquals(minDataSet.sub(theoreticalMin).max(1).getDouble(0), 0.0, 0.000000001);
    assertEquals(maxDataSet.sub(theoreticalMax).max(1).getDouble(0), 0.0, 0.000000001);

    sampleIter.setPreProcessor(myNormalizer);
    INDArray actual, expected, delta;
    int i = 1;
    while (sampleIter.hasNext()) {
        expected = theoreticalMin.mul(i - 1).div(theoreticalRange);
        actual = sampleIter.next().getFeatures();
        delta = Transforms.abs(actual.sub(expected));
        assertTrue(delta.max(1).getDouble(0) < 0.0001);
        i++;
    }

}
 

@Test
public void testTransform() {
    /*Random dataset is generated such that
        AX + B where X is from a normal distribution with mean 0 and std 1
        The mean of above will be B and std A
        Obtained mean and std dev are compared to theoretical
        Transformed values should be the same as X with the same seed.
     */
    long randSeed = 2227724;

    int nFeatures = 2;
    int nSamples = 6400;
    int bsize = 8;
    int a = 5;
    int b = 100;
    INDArray sampleMean, sampleStd, sampleMeanDelta, sampleStdDelta, delta, deltaPerc;
    double maxDeltaPerc, sampleMeanSEM;

    genRandomDataSet normData = new genRandomDataSet(nSamples, nFeatures, a, b, randSeed);
    genRandomDataSet expectedData = new genRandomDataSet(nSamples, nFeatures, 1, 0, randSeed);
    genRandomDataSet beforeTransformData = new genRandomDataSet(nSamples, nFeatures, a, b, randSeed);

    NormalizerStandardize myNormalizer = new NormalizerStandardize();
    myNormalizer.fitLabel(true);
    DataSetIterator normIterator = normData.getIter(bsize);
    DataSetIterator expectedIterator = expectedData.getIter(bsize);
    DataSetIterator beforeTransformIterator = beforeTransformData.getIter(bsize);

    myNormalizer.fit(normIterator);

    double tolerancePerc = 0.5; //within 0.5%
    sampleMean = myNormalizer.getMean();
    sampleMeanDelta = Transforms.abs(sampleMean.sub(normData.theoreticalMean));
    assertTrue(sampleMeanDelta.mul(100).div(normData.theoreticalMean).max(1).getDouble(0, 0) < tolerancePerc);
    //sanity check to see if it's within the theoretical standard error of mean
    sampleMeanSEM = sampleMeanDelta.div(normData.theoreticalSEM).max(1).getDouble(0, 0);
    assertTrue(sampleMeanSEM < 2.6); //99% of the time it should be within this many SEMs

    tolerancePerc = 5; //within 5%
    sampleStd = myNormalizer.getStd();
    sampleStdDelta = Transforms.abs(sampleStd.sub(normData.theoreticalStd));
    assertTrue(sampleStdDelta.div(normData.theoreticalStd).max(1).mul(100).getDouble(0, 0) < tolerancePerc);

    tolerancePerc = 1; //within 1%
    normIterator.setPreProcessor(myNormalizer);
    while (normIterator.hasNext()) {
        INDArray before = beforeTransformIterator.next().getFeatures();
        DataSet here = normIterator.next();
        assertEquals(here.getFeatures(), here.getLabels()); //bootstrapping existing test on features
        INDArray after = here.getFeatures();
        INDArray expected = expectedIterator.next().getFeatures();
        delta = Transforms.abs(after.sub(expected));
        deltaPerc = delta.div(before.sub(expected));
        deltaPerc.muli(100);
        maxDeltaPerc = deltaPerc.max(0, 1).getDouble(0, 0);
        //System.out.println("=== BEFORE ===");
        //System.out.println(before);
        //System.out.println("=== AFTER ===");
        //System.out.println(after);
        //System.out.println("=== SHOULD BE ===");
        //System.out.println(expected);
        assertTrue(maxDeltaPerc < tolerancePerc);
    }
}
 

public static void main(String[] args) throws Exception {
    final int numRows = 28;
    final int numColumns = 28;
    int seed = 123;
    int numSamples = MnistDataFetcher.NUM_EXAMPLES;
    int batchSize = 1000;
    int iterations = 1;
    int listenerFreq = iterations/5;

    log.info("Load data....");
    DataSetIterator iter = new MnistDataSetIterator(batchSize,numSamples,true);

    log.info("Build model....");
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(seed)
            .iterations(iterations)
            .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT)
            .list(10)
            .layer(0, new RBM.Builder().nIn(numRows * numColumns).nOut(1000).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(1, new RBM.Builder().nIn(1000).nOut(500).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(2, new RBM.Builder().nIn(500).nOut(250).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(3, new RBM.Builder().nIn(250).nOut(100).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(4, new RBM.Builder().nIn(100).nOut(30).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build()) //encoding stops
            .layer(5, new RBM.Builder().nIn(30).nOut(100).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build()) //decoding starts
            .layer(6, new RBM.Builder().nIn(100).nOut(250).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(7, new RBM.Builder().nIn(250).nOut(500).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(8, new RBM.Builder().nIn(500).nOut(1000).lossFunction(LossFunctions.LossFunction.RMSE_XENT).build())
            .layer(9, new OutputLayer.Builder(LossFunctions.LossFunction.RMSE_XENT).nIn(1000).nOut(numRows*numColumns).build())
            .pretrain(true).backprop(true)
            .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    model.setListeners(Arrays.asList((IterationListener) new ScoreIterationListener(listenerFreq)));

    log.info("Train model....");
    while(iter.hasNext()) {
        DataSet next = iter.next();
        model.fit(new DataSet(next.getFeatureMatrix(),next.getFeatureMatrix()));
    }
}
 

@Test
public void testPreprocessedData() {
    //Test _loading_ of preprocessed data
    int dataSetObjSize = 5;
    int batchSizePerExecutor = 10;

    String path = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_testpreprocdata");
    File f = new File(path);
    if (f.exists())
        f.delete();
    f.mkdir();

    DataSetIterator iter = new IrisDataSetIterator(5, 150);
    int i = 0;
    while (iter.hasNext()) {
        File f2 = new File(FilenameUtils.concat(path, "data" + (i++) + ".bin"));
        iter.next().save(f2);
    }

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(3)
                                    .activation(Activation.TANH).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3).activation(Activation.SOFTMAX)
                                                    .build())
                    .build();

    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf,
                    new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize)
                                    .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1)
                                    .repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);

    sparkNet.fit("file:///" + path.replaceAll("\\\\", "/"));

    SparkTrainingStats sts = sparkNet.getSparkTrainingStats();
    int expNumFits = 12; //4 'fits' per averaging (4 executors, 1 averaging freq); 10 examples each -> 40 examples per fit. 150/40 = 3 averagings (round down); 3*4 = 12

    //Unfortunately: perfect partitioning isn't guaranteed by SparkUtils.balancedRandomSplit (esp. if original partitions are all size 1
    // which appears to be occurring at least some of the time), but we should get close to what we expect...
    assertTrue(Math.abs(expNumFits - sts.getValue("ParameterAveragingWorkerFitTimesMs").size()) < 3);

    assertEquals(3, sts.getValue("ParameterAveragingMasterMapPartitionsTimesMs").size());
}
 

public static void main(String[] args) throws IOException, InterruptedException {
    if(FEATURE_DIR.equals("{PATH-TO-PHYSIONET-FEATURES}") || LABEL_DIR.equals("{PATH-TO-PHYSIONET-LABELS")){
        System.out.println("Please provide proper directory path in place of: PATH-TO-PHYSIONET-FEATURES && PATH-TO-PHYSIONET-LABELS");
        throw new FileNotFoundException();
    }
    SequenceRecordReader trainFeaturesReader = new CSVSequenceRecordReader(1, ",");
    trainFeaturesReader.initialize(new NumberedFileInputSplit(FEATURE_DIR+"/%d.csv",0,3199));
    SequenceRecordReader trainLabelsReader = new CSVSequenceRecordReader();
    trainLabelsReader.initialize(new NumberedFileInputSplit(LABEL_DIR+"/%d.csv",0,3199));
    DataSetIterator trainDataSetIterator = new SequenceRecordReaderDataSetIterator(trainFeaturesReader,trainLabelsReader,100,2,false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END);

    SequenceRecordReader testFeaturesReader = new CSVSequenceRecordReader(1, ",");
    testFeaturesReader.initialize(new NumberedFileInputSplit(FEATURE_DIR+"/%d.csv",3200,3999));
    SequenceRecordReader testLabelsReader = new CSVSequenceRecordReader();
    testLabelsReader.initialize(new NumberedFileInputSplit(LABEL_DIR+"/%d.csv",3200,3999));
    DataSetIterator testDataSetIterator = new SequenceRecordReaderDataSetIterator(testFeaturesReader,testLabelsReader,100,2,false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END);

    ComputationGraphConfiguration configuration = new NeuralNetConfiguration.Builder()
                                                    .seed(RANDOM_SEED)
                                                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                                                    .weightInit(WeightInit.XAVIER)
                                                    .updater(new Adam())
                                                    .dropOut(0.9)
                                                    .graphBuilder()
                                                    .addInputs("trainFeatures")
                                                    .setOutputs("predictMortality")
                                                    .addLayer("L1", new LSTM.Builder()
                                                                                   .nIn(86)
                                                                                    .nOut(200)
                                                                                    .forgetGateBiasInit(1)
                                                                                    .activation(Activation.TANH)
                                                                                    .build(),"trainFeatures")
                                                    .addLayer("predictMortality", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                                                        .activation(Activation.SOFTMAX)
                                                                                        .nIn(200).nOut(2).build(),"L1")
                                                    .build();

    ComputationGraph model = new ComputationGraph(configuration);

    for(int i=0;i<1;i++){
       model.fit(trainDataSetIterator);
       trainDataSetIterator.reset();
    }
    ROC evaluation = new ROC(100);
    while (testDataSetIterator.hasNext()) {
        DataSet batch = testDataSetIterator.next();
        INDArray[] output = model.output(batch.getFeatures());
        evaluation.evalTimeSeries(batch.getLabels(), output[0]);
    }
    
    System.out.println(evaluation.calculateAUC());
    System.out.println(evaluation.stats());
}