Java Code Examples for org.deeplearning4j.nn.graph.ComputationGraph#init()

private ComputationGraph getGraph(int numLabels, double lambda) {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .dist(new NormalDistribution(0, 1)).updater(new NoOp())
                    .graphBuilder().addInputs("input1")
                    .addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(),
                                    "input1")
                    .addLayer("lossLayer", new CenterLossOutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(numLabels)
                                    .lambda(lambda).activation(Activation.SOFTMAX).build(), "l1")
                    .setOutputs("lossLayer").build();

    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();

    return graph;
}
 

@Test
public void testUICompGraph() {

    StatsStorage ss = new InMemoryStatsStorage();

    UIServer uiServer = UIServer.getInstance();
    uiServer.attach(ss);

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
                    .addLayer("L0", new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(4).build(),
                                    "in")
                    .addLayer("L1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(4).nOut(3).build(), "L0")
                    .setOutputs("L1").build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    net.setListeners(new StatsListener(ss), new ScoreIterationListener(1));

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    for (int i = 0; i < 100; i++) {
        net.fit(iter);
    }
}
 

/**
 * Test identity mapping for 2d convolution
 */
@Test
public void testIdConv2D() {
    final INDArray input = Nd4j.randn(DataType.FLOAT,1,5,7,11);
    final String inputName = "input";
    final String conv = "conv";
    final String output = "output";
    final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder()
            .graphBuilder()
            .setInputTypes(InputType.inferInputType(input))
            .addInputs(inputName)
            .setOutputs(output)
            .layer(conv, new ConvolutionLayer.Builder(3,5)
                    .convolutionMode(ConvolutionMode.Same)
                    .nOut(input.size(1))
                    .weightInit(new WeightInitIdentity())
                    .activation(new ActivationIdentity())
                    .build(), inputName)
            .layer(output, new CnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv)
            .build());
    graph.init();

    assertEquals("Mapping was not identity!", input, graph.outputSingle(input));
}
 

@Override
public ComputationGraph init() {
    ComputationGraphConfiguration.GraphBuilder graph = graphBuilder();
    ComputationGraphConfiguration conf = graph.build();
    ComputationGraph model = new ComputationGraph(conf);
    model.init();

    return model;
}
 

@Test
public void testElementWiseVertexForwardSubtract() {
    int batchsz = 24;
    int featuresz = 17;
    ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder()
                    .addInputs("input1", "input2")
                    .addLayer("denselayer",
                                    new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY)
                                                    .build(),
                                    "input1")
                    /* denselayer is not actually used, but it seems that you _need_ to have trainable parameters, otherwise, you get
                     * Invalid shape: Requested INDArray shape [1, 0] contains dimension size values < 1 (all dimensions must be 1 or more)
                     * at org.nd4j.linalg.factory.Nd4j.checkShapeValues(Nd4j.java:4877)
                     * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4867)
                     * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4820)
                     * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:3948)
                     * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:409)
                     * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:341)
                     */
                    .addVertex("elementwiseSubtract", new ElementWiseVertex(ElementWiseVertex.Op.Subtract),
                                    "input1", "input2")
                    .addLayer("Subtract", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(),
                                    "elementwiseSubtract")
                    .setOutputs("Subtract", "denselayer").build();

    ComputationGraph cg = new ComputationGraph(cgc);
    cg.init();


    INDArray input1 = Nd4j.rand(batchsz, featuresz);
    INDArray input2 = Nd4j.rand(batchsz, featuresz);

    INDArray target = input1.dup().subi(input2);

    INDArray output = cg.output(input1, input2)[0];
    INDArray squared = output.sub(target);
    double rms = Math.sqrt(squared.mul(squared).sumNumber().doubleValue());
    Assert.assertEquals(0.0, rms, this.epsilon);
}
 

public static void main(String[] args) throws Exception {
    //Define a simple ComputationGraph:
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
        .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9))
        .graphBuilder()
        .addInputs("in")
        .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.TANH).build(), "in")
        .addLayer("layer1", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0")
        .setOutputs("layer1")
        .backprop(true).pretrain(false).build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();


    //Save the model
    File locationToSave = new File("model/MyComputationGraph.zip");       //Where to save the network. Note: the file is in .zip format - can be opened externally
    boolean saveUpdater = true;                                             //Updater: i.e., the state for Momentum, RMSProp, Adagrad etc. Save this if you want to train your network more in the future
    ModelSerializer.writeModel(net, locationToSave, saveUpdater);

    //Load the model
    ComputationGraph restored = ModelSerializer.restoreComputationGraph(locationToSave);


    System.out.println("Saved and loaded parameters are equal:      " + net.params().equals(restored.params()));
    System.out.println("Saved and loaded configurations are equal:  " + net.getConfiguration().equals(restored.getConfiguration()));
}
 

@Test
public void testChangeNOutNIn() {
    final String inputName = "input";
    final String changeNoutName = "changeNout";
    final String poolName = "pool";
    final String afterPoolName = "afterPool";
    final String outputName = "output";
    final INDArray input = Nd4j.create(new long[] {1, 2, 4, 4});
    final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder()
            .graphBuilder()
            .addInputs(inputName)
            .setOutputs(outputName)
            .setInputTypes(InputType.inferInputTypes(input))
            .addLayer(changeNoutName, new Convolution2D.Builder(1, 1)
                    .nOut(10)
                    .build(), inputName)
            .addLayer(poolName, new SubsamplingLayer.Builder(1,1).build(), changeNoutName)
            .addLayer(afterPoolName, new Convolution2D.Builder(1, 1)
                    .nOut(7)
                    .build(), poolName)
            .addLayer(outputName, new OutputLayer.Builder()
                    .activation(Activation.SOFTMAX)
                    .nOut(2)
                    .build(), afterPoolName)
            .build());
    graph.init();

    final ComputationGraph newGraph = new TransferLearning.GraphBuilder(graph)
            .nOutReplace(changeNoutName, 5, WeightInit.XAVIER)
            .nInReplace(afterPoolName, 5, WeightInit.XAVIER)
            .build();

    newGraph.init();

    assertEquals("Incorrect number of outputs!", 5 , newGraph.layerSize(changeNoutName));
    assertEquals("Incorrect number of inputs!", 5, newGraph.layerInputSize(afterPoolName));
    newGraph.output(input);
}
 

@Test
    public void testAddOutput() {
        NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.9))
                        .activation(Activation.IDENTITY);

        ComputationGraphConfiguration conf = overallConf.graphBuilder().addInputs("inCentre", "inRight")
                        .addLayer("denseCentre0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "inCentre")
                        .addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "inRight")
                        .addVertex("mergeRight", new MergeVertex(), "denseCentre0", "denseRight0")
                        .addLayer("outRight",
                                        new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(4).nOut(2).build(),
                                        "mergeRight")
                        .setOutputs("outRight").build();
        ComputationGraph modelToTune = new ComputationGraph(conf);
        modelToTune.init();

        ComputationGraph modelNow =
                        new TransferLearning.GraphBuilder(modelToTune)
                                        .addLayer("outCentre",
                                                        new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(2)
                                                                        .nOut(3).build(),
                                                        "denseCentre0")
                                        .setOutputs("outRight", "outCentre").build();

        assertEquals(2, modelNow.getNumOutputArrays());
        MultiDataSet rand = new MultiDataSet(new INDArray[] {Nd4j.rand(2, 2), Nd4j.rand(2, 2)},
                        new INDArray[] {Nd4j.rand(2, 2), Nd4j.rand(2, 3)});
        modelNow.fit(rand);
//        log.info(modelNow.summary());
//        log.info(modelNow.summary(InputType.feedForward(2),InputType.feedForward(2)));
        modelNow.summary();
        modelNow.summary(InputType.feedForward(2),InputType.feedForward(2));
    }
 

@Test
public void testDuplicateToTimeSeriesVertex() {

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder()
                    .addInputs("in2d", "in3d")
                    .addVertex("duplicateTS", new DuplicateToTimeSeriesVertex("in3d"), "in2d")
                    .addLayer("out", new OutputLayer.Builder().nIn(1).nOut(1).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build(), "duplicateTS")
                    .addLayer("out3d", new RnnOutputLayer.Builder().nIn(1).nOut(1).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build(), "in3d")
                    .setOutputs("out", "out3d").build();

    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();

    INDArray in2d = Nd4j.rand(3, 5);
    INDArray in3d = Nd4j.rand(new int[] {3, 2, 7});

    graph.setInputs(in2d, in3d);

    INDArray expOut = Nd4j.zeros(3, 5, 7);
    for (int i = 0; i < 7; i++) {
        expOut.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(i)}, in2d);
    }

    GraphVertex gv = graph.getVertex("duplicateTS");
    gv.setInputs(in2d);
    INDArray outFwd = gv.doForward(true, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expOut, outFwd);

    INDArray expOutBackward = expOut.sum(2);
    gv.setEpsilon(expOut);
    INDArray outBwd = gv.doBackward(false, LayerWorkspaceMgr.noWorkspaces()).getSecond()[0];
    assertEquals(expOutBackward, outBwd);

    String json = conf.toJson();
    ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json);
    assertEquals(conf, conf2);
}
 

@Test
    public void testBasicStackUnstackDebug() {
        Nd4j.getRandom().setSeed(12345);

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                        .dataType(DataType.DOUBLE)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .dist(new NormalDistribution(0, 1))
                        .activation(Activation.TANH).updater(new NoOp()).graphBuilder()
                        .addInputs("in1", "in2").addLayer("d0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "in1")
                        .addLayer("d1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "in2")
                        .addVertex("stack", new StackVertex(), "d0", "d1")
                        .addVertex("u0", new UnstackVertex(0, 2), "stack")
                        .addVertex("u1", new UnstackVertex(1, 2), "stack")
                        .addLayer("out1",
                                        new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.L2).nIn(2)
                                                        .nOut(2).activation(Activation.IDENTITY).build(),
                                        "u0")
                        .addLayer("out2",
                                        new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.L2).nIn(2)
                                                        .nOut(2).activation(Activation.IDENTITY).build(),
                                        "u1")
                        .setOutputs("out1", "out2").build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();


        Nd4j.getRandom().setSeed(12345);
        long nParams = graph.numParams();
        INDArray newParams = Nd4j.rand(new long[]{1, nParams});
        graph.setParams(newParams);

        int[] mbSizes = new int[] {1, 3, 10};
        for (int minibatch : mbSizes) {

            INDArray in1 = Nd4j.rand(minibatch, 2);
            INDArray in2 = Nd4j.rand(minibatch, 2);

            INDArray labels1 = Nd4j.rand(minibatch, 2);
            INDArray labels2 = Nd4j.rand(minibatch, 2);

            String testName = "testBasicStackUnstack() - minibatch = " + minibatch;

            if (PRINT_RESULTS) {
                System.out.println(testName);
//                for (int j = 0; j < graph.getNumLayers(); j++)
//                    System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{in1, in2})
                    .labels(new INDArray[]{labels1, labels2}));

            assertTrue(testName, gradOK);
            TestUtils.testModelSerialization(graph);
        }
    }
 

@Test
public void testCnnLossLayerCompGraph(){

    for(WorkspaceMode ws : WorkspaceMode.values()) {
        log.info("*** Testing workspace: " + ws);

        for (Activation a : new Activation[]{Activation.TANH, Activation.SELU}) {
            //Check that (A+identity) is equal to (identity+A), for activation A
            //i.e., should get same output and weight gradients for both

            ComputationGraphConfiguration conf1 =
                    new NeuralNetConfiguration.Builder().seed(12345L)
                            .updater(new NoOp())
                            .convolutionMode(ConvolutionMode.Same)
                            .inferenceWorkspaceMode(ws)
                            .trainingWorkspaceMode(ws)
                            .graphBuilder()
                            .addInputs("in")
                            .addLayer("0", new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY)
                                    .kernelSize(2, 2).stride(1, 1)
                                    .dist(new NormalDistribution(0, 1.0))
                                    .updater(new NoOp()).build(), "in")
                            .addLayer("1", new CnnLossLayer.Builder(LossFunction.MSE)
                                    .activation(a)
                                    .build(), "0")
                            .setOutputs("1")
                            .build();

            ComputationGraphConfiguration conf2 =
                    new NeuralNetConfiguration.Builder().seed(12345L)
                            .updater(new NoOp())
                            .convolutionMode(ConvolutionMode.Same)
                            .inferenceWorkspaceMode(ws)
                            .trainingWorkspaceMode(ws)
                            .graphBuilder()
                            .addInputs("in")
                            .addLayer("0", new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(a)
                                    .kernelSize(2, 2).stride(1, 1)
                                    .dist(new NormalDistribution(0, 1.0))
                                    .updater(new NoOp()).build(), "in")
                            .addLayer("1", new CnnLossLayer.Builder(LossFunction.MSE)
                                    .activation(Activation.IDENTITY)
                                    .build(), "0")
                            .setOutputs("1")
                            .build();

            ComputationGraph graph = new ComputationGraph(conf1);
            graph.init();

            ComputationGraph graph2 = new ComputationGraph(conf2);
            graph2.init();


            graph2.setParams(graph.params());


            INDArray in = Nd4j.rand(new int[]{3, 3, 5, 5});

            INDArray out1 = graph.outputSingle(in);
            INDArray out2 = graph2.outputSingle(in);

            assertEquals(out1, out2);

            INDArray labels = Nd4j.rand(out1.shape());

            graph.setInput(0,in);
            graph.setLabels(labels);

            graph2.setInput(0,in);
            graph2.setLabels(labels);

            graph.computeGradientAndScore();
            graph2.computeGradientAndScore();

            assertEquals(graph.score(), graph2.score(), 1e-6);
            assertEquals(graph.gradient().gradient(), graph2.gradient().gradient());

            //Also check computeScoreForExamples
            INDArray in2a = Nd4j.rand(new int[]{1, 3, 5, 5});
            INDArray labels2a = Nd4j.rand(new int[]{1, 4, 5, 5});

            INDArray in2 = Nd4j.concat(0, in2a, in2a);
            INDArray labels2 = Nd4j.concat(0, labels2a, labels2a);

            INDArray s = graph.scoreExamples(new DataSet(in2, labels2), false);
            assertArrayEquals(new long[]{2, 1}, s.shape());
            assertEquals(s.getDouble(0), s.getDouble(1), 1e-6);

            TestUtils.testModelSerialization(graph);
        }
    }
}
 

@Test
    public void testRNNWithMerging() {

        for(RNNFormat format : RNNFormat.values()) {

            String msg = "testLSTMWithMerging - " + format;

            Nd4j.getRandom().setSeed(12345);
            ComputationGraphConfiguration conf =
                    new NeuralNetConfiguration.Builder().seed(12345)
                            .dataType(DataType.DOUBLE)
                            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                            .dist(new UniformDistribution(0.2, 0.6))
                            .updater(new NoOp()).graphBuilder().addInputs("input")
                            .setOutputs("out")
                            .addLayer("lstm1",
                                    new SimpleRnn.Builder().nIn(3).nOut(3)
                                            .activation(Activation.TANH).build(),
                                    "input")
                            .addLayer("lstm2",
                                    new SimpleRnn.Builder().nIn(3).nOut(3)
                                            .activation(Activation.TANH).build(),
                                    "lstm1")
                            .addLayer("dense1",
                                    new DenseLayer.Builder().nIn(3).nOut(3)
                                            .activation(Activation.SIGMOID).build(),
                                    "lstm1")
                            .addLayer("lstm3",
                                    new SimpleRnn.Builder().nIn(3).nOut(3)
                                            .activation(Activation.TANH).build(),
                            "dense1")
                            .addVertex("merge", new MergeVertex(), "lstm2", "lstm3")
                            .addLayer("out", new RnnOutputLayer.Builder().nIn(6).nOut(3)
                                            .activation(Activation.SOFTMAX)
                                            .lossFunction(LossFunctions.LossFunction.MCXENT).build(),
                                    "merge")
                            .setInputTypes(InputType.recurrent(4, format))
                            .build();

            ComputationGraph graph = new ComputationGraph(conf);
            graph.init();

            Random r = new Random(12345);
            INDArray input = Nd4j.rand(DataType.DOUBLE, format == RNNFormat.NCW ? new long[]{2, 3, 4} : new long[]{2,4,3});
            INDArray labels = TestUtils.randomOneHotTimeSeries(format, 2, 3, 4, new Random(12345));

            if (PRINT_RESULTS) {
                System.out.println(msg);
//            for (int j = 0; j < graph.getNumLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                    .labels(new INDArray[]{labels}));

            assertTrue(msg, gradOK);
            TestUtils.testModelSerialization(graph);

        }
    }
 

@Test
public void testModelSerializerFrozenLayersCompGraph() throws Exception {
    FineTuneConfiguration finetune = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build();

    int nIn = 6;
    int nOut = 3;

    ComputationGraphConfiguration origConf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).graphBuilder().addInputs("in")
                    .addLayer("0", new DenseLayer.Builder().nIn(nIn).nOut(5).build(), "in")
                    .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(4).build(), "0")
                    .addLayer("2", new DenseLayer.Builder().nIn(4).nOut(3).build(), "1")
                    .addLayer("3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3)
                                                    .nOut(nOut).build(),
                                    "2")
                    .setOutputs("3").build();
    ComputationGraph origModel = new ComputationGraph(origConf);
    origModel.init();

    ComputationGraph withFrozen = new TransferLearning.GraphBuilder(origModel).fineTuneConfiguration(finetune)
                    .setFeatureExtractor("1").build();

    assertTrue(withFrozen.getLayer(0) instanceof FrozenLayer);
    assertTrue(withFrozen.getLayer(1) instanceof FrozenLayer);

    Map<String, GraphVertex> m = withFrozen.getConfiguration().getVertices();
    Layer l0 = ((LayerVertex) m.get("0")).getLayerConf().getLayer();
    Layer l1 = ((LayerVertex) m.get("1")).getLayerConf().getLayer();
    assertTrue(l0 instanceof org.deeplearning4j.nn.conf.layers.misc.FrozenLayer);
    assertTrue(l1 instanceof org.deeplearning4j.nn.conf.layers.misc.FrozenLayer);

    ComputationGraph restored = TestUtils.testModelSerialization(withFrozen);

    assertTrue(restored.getLayer(0) instanceof FrozenLayer);
    assertTrue(restored.getLayer(1) instanceof FrozenLayer);
    assertFalse(restored.getLayer(2) instanceof FrozenLayer);
    assertFalse(restored.getLayer(3) instanceof FrozenLayer);

    INDArray in = Nd4j.rand(3, nIn);
    INDArray out = withFrozen.outputSingle(in);
    INDArray out2 = restored.outputSingle(in);

    assertEquals(out, out2);

    //Sanity check on train mode:
    out = withFrozen.outputSingle(true, in);
    out2 = restored.outputSingle(true, in);
}
 

@Test
    public void testTransferGlobalPool() {

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(0.1))
                .weightInit(WeightInit.XAVIER)
                        .graphBuilder().addInputs("in")
                        .addLayer("blstm1",new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10)
                                                        .activation(Activation.TANH).build(),
                                        "in")
                        .addLayer("pool", new GlobalPoolingLayer.Builder().build(), "blstm1")
                        .addLayer("dense", new DenseLayer.Builder().nIn(10).nOut(10).build(), "pool")
                        .addLayer("out", new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.IDENTITY)
                                        .lossFunction(LossFunctions.LossFunction.MSE).build(), "dense")
                        .setOutputs("out").build();

        ComputationGraph g = new ComputationGraph(conf);
        g.init();

        FineTuneConfiguration fineTuneConfiguration =
                        new FineTuneConfiguration.Builder().seed(12345).updater(new Sgd(0.01)).build();

        ComputationGraph graph = new TransferLearning.GraphBuilder(g).fineTuneConfiguration(fineTuneConfiguration)
                        .removeVertexKeepConnections("out").setFeatureExtractor("dense")
                        .addLayer("out", new OutputLayer.Builder().updater(new Adam(0.1))
                                .weightInit(WeightInit.XAVIER)
                                        .activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT)
                                        .nIn(10).nOut(5).build(), "dense")
                        .build();

        ComputationGraphConfiguration confExpected = new NeuralNetConfiguration.Builder().seed(12345)
                        .updater(new Sgd(0.01))
                        .weightInit(WeightInit.XAVIER)
                        .graphBuilder().addInputs("in")
                        .addLayer("blstm1",
                                        new FrozenLayer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10)
                                                        .activation(Activation.TANH).build()),
                                        "in")
                        .addLayer("pool", new FrozenLayer(new GlobalPoolingLayer.Builder().build()), "blstm1")
                        .addLayer("dense", new FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).build()), "pool")
                        .addLayer("out", new OutputLayer.Builder().nIn(10).nOut(5).activation(Activation.SOFTMAX)
                                        .updater(new Adam(0.1))
                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "dense")
                        .setOutputs("out").build();

        ComputationGraph modelExpected = new ComputationGraph(confExpected);
        modelExpected.init();


//        assertEquals(confExpected, graph.getConfiguration());
        assertEquals(confExpected.toJson(), graph.getConfiguration().toJson());
    }
 

@Test
public void testPutGetObject() throws Exception {

    int nIn = 5;
    int nOut = 6;

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01)
            .graphBuilder()
            .addInputs("in")
            .layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build(), "in")
            .setOutputs("0")
            .build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    File tempFile = tempDir.newFile();
    ModelSerializer.writeModel(net, tempFile, true);


    List<String> toWrite = Arrays.asList("zero", "one", "two");
    ModelSerializer.addObjectToFile(tempFile, "myLabels", toWrite);
    List<String> restored = ModelSerializer.getObjectFromFile(tempFile, "myLabels");
    assertEquals(toWrite, restored);


    Map<String,Object> someOtherData = new HashMap<>();
    someOtherData.put("x", new float[]{0,1,2});
    someOtherData.put("y",Nd4j.linspace(1,10,10, Nd4j.dataType()));

    ModelSerializer.addObjectToFile(tempFile, "otherData.bin", someOtherData);

    Map<String,Object> dataRestored = ModelSerializer.getObjectFromFile(tempFile, "otherData.bin");
    assertEquals(someOtherData.keySet(), dataRestored.keySet());
    assertArrayEquals((float[])someOtherData.get("x"), (float[])dataRestored.get("x"), 0f);
    assertEquals(someOtherData.get("y"), dataRestored.get("y"));


    List<String> entries = ModelSerializer.listObjectsInFile(tempFile);
    assertEquals(2, entries.size());
    System.out.println(entries);
    assertTrue(entries.contains("myLabels"));
    assertTrue(entries.contains("otherData.bin"));

    ComputationGraph restoredNet = ModelSerializer.restoreComputationGraph(tempFile);
    assertEquals(net.params(), restoredNet.params());
}
 

@Test
public void testSameDiffLamdaVertexBasic(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        log.info("--- Workspace Mode: {} ---", wsm);

        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("lambda", new SameDiffSimpleLambdaVertex(), "0", "1")
                .addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lambda")
                .setOutputs("2")
                .build();

        //Equavalent, not using SameDiff Lambda:
        ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("elementwise", new ElementWiseVertex(ElementWiseVertex.Op.Product), "0", "1")
                .addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "elementwise")
                .setOutputs("3")
                .build();

        ComputationGraph lambda = new ComputationGraph(conf);
        lambda.init();

        ComputationGraph std = new ComputationGraph(confStd);
        std.init();

        lambda.setParams(std.params());

        INDArray in1 = Nd4j.rand(3, 5);
        INDArray in2 = Nd4j.rand(3, 5);
        INDArray labels = TestUtils.randomOneHot(3, 5);
        MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{in1, in2}, new INDArray[]{labels});

        INDArray outLambda = lambda.output(in1, in2)[0];
        INDArray outStd = std.output(in1, in2)[0];

        assertEquals(outLambda, outStd);

        double scoreLambda = lambda.score(mds);
        double scoreStd = std.score(mds);

        assertEquals(scoreStd, scoreLambda, 1e-6);

        for (int i = 0; i < 3; i++) {
            lambda.fit(mds);
            std.fit(mds);

            String s = String.valueOf(i);
            assertEquals(s, std.params(), lambda.params());
            assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
        }

        ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
        outLambda = loaded.output(in1, in2)[0];
        outStd = std.output(in1, in2)[0];

        assertEquals(outStd, outLambda);

        //Sanity check on different minibatch sizes:
        INDArray newIn1 = Nd4j.vstack(in1, in1);
        INDArray newIn2 = Nd4j.vstack(in2, in2);
        INDArray outMbsd = lambda.output(newIn1, newIn2)[0];
        INDArray outMb = std.output(newIn1, newIn2)[0];
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void testFrozenLayerInstantiationCompGraph() {

    //We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if
    // they were initialized via the builder
    ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder()
            .addInputs("in")
            .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build(), "in")
            .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build(), "0")
            .addLayer("2", new OutputLayer.Builder(
                            LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10)
                            .nOut(10).build(),
                    "1")
            .setOutputs("2").build();

    ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder()
            .addInputs("in")
            .addLayer("0", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(
                    new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                            .weightInit(WeightInit.XAVIER).build()), "in")
            .addLayer("1", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(
                    new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                            .weightInit(WeightInit.XAVIER).build()), "0")
            .addLayer("2", new OutputLayer.Builder(
                            LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10)
                            .nOut(10).build(),
                    "1")
            .setOutputs("2").build();

    ComputationGraph net1 = new ComputationGraph(conf1);
    net1.init();
    ComputationGraph net2 = new ComputationGraph(conf2);
    net2.init();

    assertEquals(net1.params(), net2.params());


    String json = conf2.toJson();
    ComputationGraphConfiguration fromJson = ComputationGraphConfiguration.fromJson(json);

    assertEquals(conf2, fromJson);

    ComputationGraph net3 = new ComputationGraph(fromJson);
    net3.init();

    INDArray input = Nd4j.rand(10, 10);

    INDArray out2 = net2.outputSingle(input);
    INDArray out3 = net3.outputSingle(input);

    assertEquals(out2, out3);
}
 

@Test
public void testGradientApplyComputationGraph() {
    int minibatch = 7;
    int nIn = 10;
    int nOut = 10;

    for (boolean regularization : new boolean[] {false, true}) {
        for (IUpdater u : new IUpdater[] {new Sgd(0.1), new Adam(0.1)}) {

            ComputationGraphConfiguration conf =
                            new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH)
                                            .weightInit(WeightInit.XAVIER).updater(u)
                                            .l1(regularization ? 0.2 : 0.0)
                                            .l2(regularization ? 0.3 : 0.0).graphBuilder().addInputs("in")
                                            .addLayer("0", new DenseLayer.Builder().nIn(nIn).nOut(10).build(), "in")
                                            .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).build(), "0")
                                            .addLayer("2", new OutputLayer.Builder(
                                                            LossFunctions.LossFunction.MCXENT)
                                                                            .activation(Activation.SOFTMAX).nIn(10)
                                                                            .nOut(nOut).build(),
                                                            "1")
                                            .setOutputs("2").build();


            Nd4j.getRandom().setSeed(12345);
            ComputationGraph net1GradCalc = new ComputationGraph(conf);
            net1GradCalc.init();

            Nd4j.getRandom().setSeed(12345);
            ComputationGraph net2GradUpd = new ComputationGraph(conf.clone());
            net2GradUpd.init();

            assertEquals(net1GradCalc.params(), net2GradUpd.params());

            INDArray f = Nd4j.rand(minibatch, nIn);
            INDArray l = Nd4j.create(minibatch, nOut);
            for (int i = 0; i < minibatch; i++) {
                l.putScalar(i, i % nOut, 1.0);
            }
            net1GradCalc.setInputs(f);
            net1GradCalc.setLabels(l);

            net2GradUpd.setInputs(f);
            net2GradUpd.setLabels(l);

            //Calculate gradient in first net, update and apply it in the second
            //Also: calculate gradient in the second net, just to be sure it isn't modified while doing updating on
            // the other net's gradient
            net1GradCalc.computeGradientAndScore();
            net2GradUpd.computeGradientAndScore();

            Gradient g = net1GradCalc.gradient();
            INDArray gBefore = g.gradient().dup(); //Net 1 gradient should be modified
            INDArray net2GradBefore = net2GradUpd.gradient().gradient().dup(); //But net 2 gradient should not be
            net2GradUpd.getUpdater().update(g, 0, 0, minibatch, LayerWorkspaceMgr.noWorkspaces());
            INDArray gAfter = g.gradient().dup();
            INDArray net2GradAfter = net2GradUpd.gradient().gradient().dup();

            assertNotEquals(gBefore, gAfter); //Net 1 gradient should be modified
            assertEquals(net2GradBefore, net2GradAfter); //But net 2 gradient should not be


            //Also: if we apply the gradient using a subi op, we should get the same final params as if we did a fit op
            // on the original network
            net2GradUpd.params().subi(g.gradient());

            net1GradCalc.fit(new INDArray[] {f}, new INDArray[] {l});
            assertEquals(net1GradCalc.params(), net2GradUpd.params());

            //=============================
            if (!(u instanceof Sgd)) {
                net2GradUpd.getUpdater().getStateViewArray().assign(net1GradCalc.getUpdater().getStateViewArray());
            }
            assertEquals(net1GradCalc.params(), net2GradUpd.params());
            assertEquals(net1GradCalc.getUpdater().getStateViewArray(),
                            net2GradUpd.getUpdater().getStateViewArray());

            //Remove the next 2 lines: fails - as net 1 is 1 iteration ahead
            net1GradCalc.getConfiguration().setIterationCount(0);
            net2GradUpd.getConfiguration().setIterationCount(0);


            for (int i = 0; i < 100; i++) {
                net1GradCalc.fit(new INDArray[] {f}, new INDArray[] {l});
                net2GradUpd.fit(new INDArray[] {f}, new INDArray[] {l});
                assertEquals(net1GradCalc.params(), net2GradUpd.params());
            }
        }
    }
}
 

@Test
    public void testFitUnFrozen() {

        NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.9)).seed(124)
                        .activation(Activation.IDENTITY)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);

        ComputationGraphConfiguration conf = overallConf.graphBuilder().addInputs("inCentre", "inRight")
                        .addLayer("denseCentre0", new DenseLayer.Builder().nIn(10).nOut(9).build(), "inCentre")
                        .addLayer("denseCentre1", new DenseLayer.Builder().nIn(9).nOut(8).build(), "denseCentre0")
                        .addLayer("denseCentre2", new DenseLayer.Builder().nIn(8).nOut(7).build(), "denseCentre1")
                        .addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2")
                        .addLayer("outCentre",
                                        new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7).nOut(4).build(),
                                        "denseCentre3")
                        .addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1")
                        .addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "subsetLeft")
                        .addLayer("outLeft",
                                        new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).build(),
                                        "denseLeft0")
                        .addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2")
                        .addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "inRight")
                        .addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0")
                        .addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), "mergeRight")
                        .addLayer("outRight",
                                        new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(5).build(),
                                        "denseRight1")
                        .setOutputs("outLeft", "outCentre", "outRight").build();

        ComputationGraph modelToTune = new ComputationGraph(conf);
        modelToTune.init();

        INDArray inRight = Nd4j.rand(10, 2);
        INDArray inCentre = Nd4j.rand(10, 10);
        INDArray outLeft = Nd4j.rand(10, 6);
        INDArray outRight = Nd4j.rand(10, 5);
        INDArray outCentre = Nd4j.rand(10, 4);
        MultiDataSet origData = new MultiDataSet(new INDArray[] {inCentre, inRight},
                        new INDArray[] {outLeft, outCentre, outRight});
        ComputationGraph modelIdentical = modelToTune.clone();
        modelIdentical.getVertex("denseCentre0").setLayerAsFrozen();
        modelIdentical.getVertex("denseCentre1").setLayerAsFrozen();
        modelIdentical.getVertex("denseCentre2").setLayerAsFrozen();

        TransferLearningHelper helper = new TransferLearningHelper(modelToTune, "denseCentre2");
        MultiDataSet featurizedDataSet = helper.featurize(origData);

        assertEquals(modelIdentical.getLayer("denseRight0").params(), modelToTune.getLayer("denseRight0").params());
        modelIdentical.fit(origData);
        helper.fitFeaturized(featurizedDataSet);

        assertEquals(modelIdentical.getLayer("denseCentre0").params(), modelToTune.getLayer("denseCentre0").params());
        assertEquals(modelIdentical.getLayer("denseCentre1").params(), modelToTune.getLayer("denseCentre1").params());
        assertEquals(modelIdentical.getLayer("denseCentre2").params(), modelToTune.getLayer("denseCentre2").params());
        assertEquals(modelIdentical.getLayer("denseCentre3").params(), modelToTune.getLayer("denseCentre3").params());
        assertEquals(modelIdentical.getLayer("outCentre").params(), modelToTune.getLayer("outCentre").params());
        assertEquals(modelIdentical.getLayer("denseRight").conf().toJson(),
                        modelToTune.getLayer("denseRight").conf().toJson());
        assertEquals(modelIdentical.getLayer("denseRight").params(), modelToTune.getLayer("denseRight").params());
        assertEquals(modelIdentical.getLayer("denseRight0").conf().toJson(),
                        modelToTune.getLayer("denseRight0").conf().toJson());
        //assertEquals(modelIdentical.getLayer("denseRight0").params(),modelToTune.getLayer("denseRight0").params());
        assertEquals(modelIdentical.getLayer("denseRight1").params(), modelToTune.getLayer("denseRight1").params());
        assertEquals(modelIdentical.getLayer("outRight").params(), modelToTune.getLayer("outRight").params());
        assertEquals(modelIdentical.getLayer("denseLeft0").params(), modelToTune.getLayer("denseLeft0").params());
        assertEquals(modelIdentical.getLayer("outLeft").params(), modelToTune.getLayer("outLeft").params());

//        log.info(modelIdentical.summary());
//        log.info(helper.unfrozenGraph().summary());
        modelIdentical.summary();
        helper.unfrozenGraph().summary();
    }
 

@Test
public void testFrozenCompGraph() {
    DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3));

    NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1))
                    .activation(Activation.IDENTITY);

    ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In")
                    .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In")
                    .addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0")
                    .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1")
                    .addLayer("layer3",
                                    new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                                    LossFunctions.LossFunction.MCXENT)
                                                                    .activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                                    .build(),
                                    "layer2")
                    .setOutputs("layer3").build());

    modelToFineTune.init();
    INDArray asFrozenFeatures = modelToFineTune.feedForward(randomData.getFeatures(), false).get("layer1");

    ComputationGraph modelNow =
                    new TransferLearning.GraphBuilder(modelToFineTune).setFeatureExtractor("layer1").build();

    ComputationGraph notFrozen = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In")
                    .addLayer("layer0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer0In")
                    .addLayer("layer1",
                                    new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                                    LossFunctions.LossFunction.MCXENT)
                                                                    .activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                                    .build(),
                                    "layer0")
                    .setOutputs("layer1").build());

    notFrozen.init();
    notFrozen.setParams(Nd4j.hstack(modelToFineTune.getLayer("layer2").params(),
                    modelToFineTune.getLayer("layer3").params()));

    int i = 0;
    while (i < 5) {
        notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels()));
        modelNow.fit(randomData);
        i++;
    }

    assertEquals(Nd4j.hstack(modelToFineTune.getLayer("layer0").params(),
                    modelToFineTune.getLayer("layer1").params(), notFrozen.params()), modelNow.params());
}