Java Code Examples for org.deeplearning4j.nn.conf.ComputationGraphConfiguration#clone()

public SparkComputationGraph(JavaSparkContext sparkContext, ComputationGraphConfiguration conf,
                TrainingMaster trainingMaster) {
    sc = sparkContext;
    this.trainingMaster = trainingMaster;
    this.conf = conf.clone();
    this.network = new ComputationGraph(conf);
    this.network.init();

    //Check if kryo configuration is correct:
    SparkUtils.checkKryoConfiguration(sparkContext, log);
}
 

@Test
@Ignore    //Ignored 2019/04/09 - low priority: https://github.com/deeplearning4j/deeplearning4j/issues/6656
public void testVaePretrainSimpleCG() {
    //Simple sanity check on pretraining
    int nIn = 8;

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp())
                    .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                    .addLayer("0", new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12)
                                    .decoderLayerSizes(13).reconstructionDistribution(
                                                    new GaussianReconstructionDistribution(Activation.IDENTITY))
                                    .build(), "in")
                    .setOutputs("0").build();

    //Do training on Spark with one executor, for 3 separate minibatches
    int rddDataSetNumExamples = 10;
    int totalAveragings = 5;
    int averagingFrequency = 3;
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples)
                    .averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples)
                    .saveUpdater(true).workerPrefetchNumBatches(0).build();
    Nd4j.getRandom().setSeed(12345);
    SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(), tm);

    List<DataSet> trainData = new ArrayList<>();
    int nDataSets = numExecutors() * totalAveragings * averagingFrequency;
    for (int i = 0; i < nDataSets; i++) {
        trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null));
    }

    JavaRDD<DataSet> data = sc.parallelize(trainData);

    sparkNet.fit(data);
}
 

@Test
public void testGradientApplyComputationGraph() {
    int minibatch = 7;
    int nIn = 10;
    int nOut = 10;

    for (boolean regularization : new boolean[] {false, true}) {
        for (IUpdater u : new IUpdater[] {new Sgd(0.1), new Adam(0.1)}) {

            ComputationGraphConfiguration conf =
                            new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH)
                                            .weightInit(WeightInit.XAVIER).updater(u)
                                            .l1(regularization ? 0.2 : 0.0)
                                            .l2(regularization ? 0.3 : 0.0).graphBuilder().addInputs("in")
                                            .addLayer("0", new DenseLayer.Builder().nIn(nIn).nOut(10).build(), "in")
                                            .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).build(), "0")
                                            .addLayer("2", new OutputLayer.Builder(
                                                            LossFunctions.LossFunction.MCXENT)
                                                                            .activation(Activation.SOFTMAX).nIn(10)
                                                                            .nOut(nOut).build(),
                                                            "1")
                                            .setOutputs("2").build();


            Nd4j.getRandom().setSeed(12345);
            ComputationGraph net1GradCalc = new ComputationGraph(conf);
            net1GradCalc.init();

            Nd4j.getRandom().setSeed(12345);
            ComputationGraph net2GradUpd = new ComputationGraph(conf.clone());
            net2GradUpd.init();

            assertEquals(net1GradCalc.params(), net2GradUpd.params());

            INDArray f = Nd4j.rand(minibatch, nIn);
            INDArray l = Nd4j.create(minibatch, nOut);
            for (int i = 0; i < minibatch; i++) {
                l.putScalar(i, i % nOut, 1.0);
            }
            net1GradCalc.setInputs(f);
            net1GradCalc.setLabels(l);

            net2GradUpd.setInputs(f);
            net2GradUpd.setLabels(l);

            //Calculate gradient in first net, update and apply it in the second
            //Also: calculate gradient in the second net, just to be sure it isn't modified while doing updating on
            // the other net's gradient
            net1GradCalc.computeGradientAndScore();
            net2GradUpd.computeGradientAndScore();

            Gradient g = net1GradCalc.gradient();
            INDArray gBefore = g.gradient().dup(); //Net 1 gradient should be modified
            INDArray net2GradBefore = net2GradUpd.gradient().gradient().dup(); //But net 2 gradient should not be
            net2GradUpd.getUpdater().update(g, 0, 0, minibatch, LayerWorkspaceMgr.noWorkspaces());
            INDArray gAfter = g.gradient().dup();
            INDArray net2GradAfter = net2GradUpd.gradient().gradient().dup();

            assertNotEquals(gBefore, gAfter); //Net 1 gradient should be modified
            assertEquals(net2GradBefore, net2GradAfter); //But net 2 gradient should not be


            //Also: if we apply the gradient using a subi op, we should get the same final params as if we did a fit op
            // on the original network
            net2GradUpd.params().subi(g.gradient());

            net1GradCalc.fit(new INDArray[] {f}, new INDArray[] {l});
            assertEquals(net1GradCalc.params(), net2GradUpd.params());

            //=============================
            if (!(u instanceof Sgd)) {
                net2GradUpd.getUpdater().getStateViewArray().assign(net1GradCalc.getUpdater().getStateViewArray());
            }
            assertEquals(net1GradCalc.params(), net2GradUpd.params());
            assertEquals(net1GradCalc.getUpdater().getStateViewArray(),
                            net2GradUpd.getUpdater().getStateViewArray());

            //Remove the next 2 lines: fails - as net 1 is 1 iteration ahead
            net1GradCalc.getConfiguration().setIterationCount(0);
            net2GradUpd.getConfiguration().setIterationCount(0);


            for (int i = 0; i < 100; i++) {
                net1GradCalc.fit(new INDArray[] {f}, new INDArray[] {l});
                net2GradUpd.fit(new INDArray[] {f}, new INDArray[] {l});
                assertEquals(net1GradCalc.params(), net2GradUpd.params());
            }
        }
    }
}
 

@Test
    public void testIterationCountsGraph() throws Exception {
        int dataSetObjSize = 5;
        int batchSizePerExecutor = 25;
        List<DataSet> list = new ArrayList<>();
        int minibatchesPerWorkerPerEpoch = 10;
        DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize,
                        batchSizePerExecutor * numExecutors() * minibatchesPerWorkerPerEpoch, false);
        while (iter.hasNext()) {
            list.add(iter.next());
        }

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp())
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .graphBuilder().addInputs("in")
                        .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50)
                                        .activation(Activation.TANH).build(), "in")
                        .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                        LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10)
                                                        .activation(Activation.SOFTMAX).build(),
                                        "0")
                        .setOutputs("1").build();

        for (int avgFreq : new int[] {1, 5, 10}) {
//            System.out.println("--- Avg freq " + avgFreq + " ---");
            SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(),
                            new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize)
                                            .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(avgFreq)
                                            .repartionData(Repartition.Always).build());

            sparkNet.setListeners(new ScoreIterationListener(5));

            JavaRDD<DataSet> rdd = sc.parallelize(list);

            assertEquals(0, sparkNet.getNetwork().getConfiguration().getIterationCount());
            sparkNet.fit(rdd);
            assertEquals(minibatchesPerWorkerPerEpoch, sparkNet.getNetwork().getConfiguration().getIterationCount());
            sparkNet.fit(rdd);
            assertEquals(2 * minibatchesPerWorkerPerEpoch,
                            sparkNet.getNetwork().getConfiguration().getIterationCount());

            sparkNet.getTrainingMaster().deleteTempFiles(sc);
        }
    }
 

@Test(timeout = 120000L)
public void testEpochCounter() throws Exception {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new OutputLayer.Builder().nIn(4).nOut(3).build())
            .build();

    ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder()
            .graphBuilder()
            .addInputs("in")
            .addLayer("out", new OutputLayer.Builder().nIn(4).nOut(3).build(), "in")
            .setOutputs("out")
            .build();

    DataSetIterator iter = new IrisDataSetIterator(1, 50);

    List<DataSet> l = new ArrayList<>();
    while(iter.hasNext()){
        l.add(iter.next());
    }

    JavaRDD<DataSet> rdd = sc.parallelize(l);


    int rddDataSetNumExamples = 1;
    int averagingFrequency = 2;
    int batch = 2;
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples)
            .averagingFrequency(averagingFrequency).batchSizePerWorker(batch)
            .saveUpdater(true).workerPrefetchNumBatches(0).build();
    Nd4j.getRandom().setSeed(12345);


    SparkDl4jMultiLayer sn1 = new SparkDl4jMultiLayer(sc, conf.clone(), tm);
    SparkComputationGraph sn2 = new SparkComputationGraph(sc, conf2.clone(), tm);


    for(int i=0; i<3; i++ ){
        assertEquals(i, sn1.getNetwork().getLayerWiseConfigurations().getEpochCount());
        assertEquals(i, sn2.getNetwork().getConfiguration().getEpochCount());
        sn1.fit(rdd);
        sn2.fit(rdd);
        assertEquals(i+1, sn1.getNetwork().getLayerWiseConfigurations().getEpochCount());
        assertEquals(i+1, sn2.getNetwork().getConfiguration().getEpochCount());
    }
}
 

@Ignore("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue")
public void testSeedRepeatability() throws Exception {

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.RMSPROP)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                    .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(4)
                                    .activation(Activation.TANH).build(), "in")
                    .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                                    .build(),
                                    "0")
                    .setOutputs("1").build();

    Nd4j.getRandom().setSeed(12345);
    ComputationGraph n1 = new ComputationGraph(conf.clone());
    n1.init();

    Nd4j.getRandom().setSeed(12345);
    ComputationGraph n2 = new ComputationGraph(conf.clone());
    n2.init();

    Nd4j.getRandom().setSeed(12345);
    ComputationGraph n3 = new ComputationGraph(conf.clone());
    n3.init();

    SparkComputationGraph sparkNet1 = new SparkComputationGraph(sc, n1,
                    new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5)
                                    .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always)
                                    .rngSeed(12345).build());

    Thread.sleep(100); //Training master IDs are only unique if they are created at least 1 ms apart...

    SparkComputationGraph sparkNet2 = new SparkComputationGraph(sc, n2,
                    new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5)
                                    .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always)
                                    .rngSeed(12345).build());

    Thread.sleep(100);

    SparkComputationGraph sparkNet3 = new SparkComputationGraph(sc, n3,
                    new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5)
                                    .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always)
                                    .rngSeed(98765).build());

    List<DataSet> data = new ArrayList<>();
    DataSetIterator iter = new IrisDataSetIterator(1, 150);
    while (iter.hasNext())
        data.add(iter.next());

    JavaRDD<DataSet> rdd = sc.parallelize(data);


    sparkNet1.fit(rdd);
    sparkNet2.fit(rdd);
    sparkNet3.fit(rdd);


    INDArray p1 = sparkNet1.getNetwork().params();
    INDArray p2 = sparkNet2.getNetwork().params();
    INDArray p3 = sparkNet3.getNetwork().params();

    sparkNet1.getTrainingMaster().deleteTempFiles(sc);
    sparkNet2.getTrainingMaster().deleteTempFiles(sc);
    sparkNet3.getTrainingMaster().deleteTempFiles(sc);

    boolean eq1 = p1.equalsWithEps(p2, 0.01);
    boolean eq2 = p1.equalsWithEps(p3, 0.01);
    assertTrue("Model 1 and 2 params should be equal", eq1);
    assertFalse("Model 1 and 3 params shoud be different", eq2);
}