org.deeplearning4j.nn.conf.layers.GravesLSTM Java Examples

public static void main(String[] args) {
    List<Data> data = readFile("");

    RegIterator trainIter = new RegIterator(data, 1, 5, 5);

    // 构建模型
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(1234)
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9))
            .list().layer(0, new GravesLSTM.Builder().activation(Activation.TANH).nIn(1).nOut(32)
                    .build())
            .layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
                    .activation(Activation.IDENTITY).nIn(32).nOut(1).build())
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.setListeners(new ScoreIterationListener(1));
    network.init();

    int epoch = 10;
    for (int i = 0; i < epoch; i++) {
        while (trainIter.hasNext()) {
            DataSet dataSets = trainIter.next();
            network.fit(dataSets);
        }
        trainIter.reset();
    }

}
 

private MultiLayerNetwork getGravesLstmNet(RNNFormat format, boolean setOnLayerAlso, boolean lastTimeStep, boolean maskZeros) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new GravesLSTM.Builder().nOut(3)
                .dataFormat(format).build(), format, lastTimeStep, maskZeros);
    } else {
        return getNetWithLayer(new GravesLSTM.Builder().nOut(3).build(), format, lastTimeStep, maskZeros);
    }
}
 

public static void main(String[] args) throws Exception {

        getModelData();
        
        System.out.println("Total memory = " + Runtime.getRuntime().totalMemory());

        int batchSize = 50;
        int vectorSize = 300;
        int nEpochs = 5;
        int truncateReviewsToLength = 300;

        MultiLayerConfiguration sentimentNN = new NeuralNetConfiguration.Builder()
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
                .updater(Updater.RMSPROP)
                .regularization(true).l2(1e-5)
                .weightInit(WeightInit.XAVIER)
                .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(1.0)
                .learningRate(0.0018)
                .list()
                .layer(0, new GravesLSTM.Builder().nIn(vectorSize).nOut(200)
                        .activation("softsign").build())
                .layer(1, new RnnOutputLayer.Builder().activation("softmax")
                        .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(200).nOut(2).build())
                .pretrain(false).backprop(true).build();

        MultiLayerNetwork net = new MultiLayerNetwork(sentimentNN);
        net.init();
        net.setListeners(new ScoreIterationListener(1));

        WordVectors wordVectors = WordVectorSerializer.loadGoogleModel(new File(GNEWS_VECTORS_PATH), true, false);
        DataSetIterator trainData = new AsyncDataSetIterator(new SentimentExampleIterator(EXTRACT_DATA_PATH, wordVectors, batchSize, truncateReviewsToLength, true), 1);
        DataSetIterator testData = new AsyncDataSetIterator(new SentimentExampleIterator(EXTRACT_DATA_PATH, wordVectors, 100, truncateReviewsToLength, false), 1);

        for (int i = 0; i < nEpochs; i++) {
            net.fit(trainData);
            trainData.reset();

            Evaluation evaluation = new Evaluation();
            while (testData.hasNext()) {
                DataSet t = testData.next();
                INDArray dataFeatures = t.getFeatureMatrix();
                INDArray dataLabels = t.getLabels();
                INDArray inMask = t.getFeaturesMaskArray();
                INDArray outMask = t.getLabelsMaskArray();
                INDArray predicted = net.output(dataFeatures, false, inMask, outMask);

                evaluation.evalTimeSeries(dataLabels, predicted, outMask);
            }
            testData.reset();

            System.out.println(evaluation.stats());
        }
    }
 

public static MultiLayerNetwork buildLstmNetworks(int nIn, int nOut) {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(seed)
            .iterations(iterations)
            .learningRate(learningRate)
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .weightInit(WeightInit.XAVIER)
            .updater(Updater.RMSPROP)
            .regularization(true)
            .l2(1e-4)
            .list()
            .layer(0, new GravesLSTM.Builder()
                    .nIn(nIn)
                    .nOut(lstmLayer1Size)
                    .activation(Activation.TANH)
                    .gateActivationFunction(Activation.HARDSIGMOID)
                    .dropOut(dropoutRatio)
                    .build())
            .layer(1, new GravesLSTM.Builder()
                    .nIn(lstmLayer1Size)
                    .nOut(lstmLayer2Size)
                    .activation(Activation.TANH)
                    .gateActivationFunction(Activation.HARDSIGMOID)
                    .dropOut(dropoutRatio)
                    .build())
            .layer(2, new DenseLayer.Builder()
            		.nIn(lstmLayer2Size)
            		.nOut(denseLayerSize)
            		.activation(Activation.RELU)
            		.build())
            .layer(3, new RnnOutputLayer.Builder()
                    .nIn(denseLayerSize)
                    .nOut(nOut)
                    .activation(Activation.IDENTITY)
                    .lossFunction(LossFunctions.LossFunction.MSE)
                    .build())
            .backpropType(BackpropType.TruncatedBPTT)
            .tBPTTForwardLength(truncatedBPTTLength)
            .tBPTTBackwardLength(truncatedBPTTLength)
            .pretrain(false)
            .backprop(true)
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    net.setListeners(new ScoreIterationListener(100));
    return net;
}
 

@Test
public void testGravesLSTM() throws Exception {

    File f = Resources.asFile("regression_testing/100a/GravesLSTMCharModelingExample_100a.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    GravesLSTM l0 = (GravesLSTM) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationTanH(), l0.getActivationFn());
    assertEquals(200, l0.getNOut());
    assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new RmsProp(0.1), l0.getIUpdater());

    GravesLSTM l1 = (GravesLSTM) net.getLayer(1).conf().getLayer();
    assertEquals(new ActivationTanH(), l1.getActivationFn());
    assertEquals(200, l1.getNOut());
    assertEquals(new WeightInitXavier(), l1.getWeightInitFn());
    assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l1));
    assertEquals(new RmsProp(0.1), l1.getIUpdater());

    RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).conf().getLayer();
    assertEquals(new ActivationSoftmax(), l2.getActivationFn());
    assertEquals(77, l2.getNOut());
    assertEquals(new WeightInitXavier(), l2.getWeightInitFn());
    assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new RmsProp(0.1), l0.getIUpdater());

    assertEquals(BackpropType.TruncatedBPTT, net.getLayerWiseConfigurations().getBackpropType());
    assertEquals(50, net.getLayerWiseConfigurations().getTbpttBackLength());
    assertEquals(50, net.getLayerWiseConfigurations().getTbpttFwdLength());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100a/GravesLSTMCharModelingExample_Output_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f2))){
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100a/GravesLSTMCharModelingExample_Input_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f3))){
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

@Test
public void testRWInit() {

    for (boolean rwInit : new boolean[]{false, true}) {
        for (int i = 0; i < 3; i++) {

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder()
                    .weightInit(new UniformDistribution(0, 1))
                    .list();

            if(rwInit) {
                switch (i) {
                    case 0:
                        b.layer(new LSTM.Builder().nIn(10).nOut(10)
                                .weightInitRecurrent(new UniformDistribution(2, 3))
                                .build());
                        break;
                    case 1:
                        b.layer(new GravesLSTM.Builder().nIn(10).nOut(10)
                                .weightInitRecurrent(new UniformDistribution(2, 3))
                                .build());
                        break;
                    case 2:
                        b.layer(new SimpleRnn.Builder().nIn(10).nOut(10)
                                .weightInitRecurrent(new UniformDistribution(2, 3)).build());
                        break;
                    default:
                        throw new RuntimeException();
                }
            } else {
                switch (i) {
                    case 0:
                        b.layer(new LSTM.Builder().nIn(10).nOut(10).build());
                        break;
                    case 1:
                        b.layer(new GravesLSTM.Builder().nIn(10).nOut(10).build());
                        break;
                    case 2:
                        b.layer(new SimpleRnn.Builder().nIn(10).nOut(10).build());
                        break;
                    default:
                        throw new RuntimeException();
                }
            }

            MultiLayerNetwork net = new MultiLayerNetwork(b.build());
            net.init();

            INDArray rw = net.getParam("0_RW");
            double min = rw.minNumber().doubleValue();
            double max = rw.maxNumber().doubleValue();
            if(rwInit){
                assertTrue(String.valueOf(min), min >= 2.0);
                assertTrue(String.valueOf(max), max <= 3.0);
            } else {
                assertTrue(String.valueOf(min), min >= 0.0);
                assertTrue(String.valueOf(max), max <= 1.0);
            }
        }
    }
}
 

@Test
    public void compareImplementationsCompGraph(){
//        for(WorkspaceMode wsm : WorkspaceMode.values()) {
        for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {
            log.info("*** Starting workspace mode: " + wsm);

            //Bidirectional(GravesLSTM) and GravesBidirectionalLSTM should be equivalent, given equivalent params
            //Note that GravesBidirectionalLSTM implements ADD mode only

            ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder()
                    .activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER)
                    .updater(new Adam())
                    .trainingWorkspaceMode(wsm)
                    .inferenceWorkspaceMode(wsm)
                    .graphBuilder()
                    .addInputs("in")
                    .layer("0", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).build()), "in")
                    .layer("1", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).build()), "0")
                    .layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE)
                            .nIn(10).nOut(10).build(), "1")
                    .setOutputs("2")
                    .build();

            ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder()
                    .activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER)
                    .updater(new Adam())
                    .trainingWorkspaceMode(wsm)
                    .inferenceWorkspaceMode(wsm)
                    .graphBuilder()
                    .addInputs("in")
                    .layer("0", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "in")
                    .layer("1", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "0")
                    .layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE)
                            .nIn(10).nOut(10).build(), "1")
                    .setOutputs("2")
                    .build();

            ComputationGraph net1 = new ComputationGraph(conf1);
            net1.init();

            ComputationGraph net2 = new ComputationGraph(conf2);
            net2.init();

            assertEquals(net1.numParams(), net2.numParams());
            for (int i = 0; i < 3; i++) {
                int n1 = (int)net1.getLayer(i).numParams();
                int n2 = (int)net2.getLayer(i).numParams();
                assertEquals(n1, n2);
            }

            net2.setParams(net1.params());  //Assuming exact same layout here...

            INDArray in = Nd4j.rand(new int[]{3, 10, 5});

            INDArray out1 = net1.outputSingle(in);
            INDArray out2 = net2.outputSingle(in);

            assertEquals(out1, out2);

            INDArray labels = Nd4j.rand(new int[]{3, 10, 5});

            net1.setInput(0,in);
            net1.setLabels(labels);

            net2.setInput(0,in);
            net2.setLabels(labels);

            net1.computeGradientAndScore();
            net2.computeGradientAndScore();

            //Ensure scores are equal:
            assertEquals(net1.score(), net2.score(), 1e-6);

            //Ensure gradients are equal:
            Gradient g1 = net1.gradient();
            Gradient g2 = net2.gradient();
            assertEquals(g1.gradient(), g2.gradient());

            //Ensure updates are equal:
            ComputationGraphUpdater u1 = (ComputationGraphUpdater) net1.getUpdater();
            ComputationGraphUpdater u2 = (ComputationGraphUpdater) net2.getUpdater();
            assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray());
            u1.update(g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces());
            u2.update(g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces());
            assertEquals(g1.gradient(), g2.gradient());
            assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray());

            //Ensure params are equal, after fitting
            net1.fit(new DataSet(in, labels));
            net2.fit(new DataSet(in, labels));

            INDArray p1 = net1.params();
            INDArray p2 = net2.params();
            assertEquals(p1, p2);
        }
    }
 

@Test
public void testSerialization() throws Exception {

    for(WorkspaceMode wsm : WorkspaceMode.values()) {
        log.info("*** Starting workspace mode: " + wsm);

        Nd4j.getRandom().setSeed(12345);

        MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                .activation(Activation.TANH)
                .weightInit(WeightInit.XAVIER)
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .updater(new Adam())
                .list()
                .layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()))
                .layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()))
                .layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE)
                        .nIn(10).nOut(10).dataFormat(rnnDataFormat).build())
                .build();

        MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
        net1.init();

        INDArray in;
        INDArray labels;

        long[] inshape = rnnDataFormat == NCW ? new long[]{3, 10, 5} : new long[]{3, 5, 10};

        in = Nd4j.rand(inshape);
        labels = Nd4j.rand(inshape);

        net1.fit(in, labels);

        byte[] bytes;
        try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
            ModelSerializer.writeModel(net1, baos, true);
            bytes = baos.toByteArray();
        }


        MultiLayerNetwork net2 = ModelSerializer.restoreMultiLayerNetwork(new ByteArrayInputStream(bytes), true);


        in = Nd4j.rand(inshape);
        labels = Nd4j.rand(inshape);

        INDArray out1 = net1.output(in);
        INDArray out2 = net2.output(in);

        assertEquals(out1, out2);

        net1.setInput(in);
        net2.setInput(in);
        net1.setLabels(labels);
        net2.setLabels(labels);

        net1.computeGradientAndScore();
        net2.computeGradientAndScore();

        assertEquals(net1.score(), net2.score(), 1e-6);
        assertEquals(net1.gradient().gradient(), net2.gradient().gradient());
    }
}
 

@Test
public void testSerializationCompGraph() throws Exception {

    for(WorkspaceMode wsm : WorkspaceMode.values()) {
        log.info("*** Starting workspace mode: " + wsm);

        Nd4j.getRandom().setSeed(12345);

        ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder()
                .activation(Activation.TANH)
                .weightInit(WeightInit.XAVIER)
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .updater(new Adam())
                .graphBuilder()
                .addInputs("in")
                .layer("0", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "in")
                .layer("1", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "0")
                .layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat)
                        .nIn(10).nOut(10).build(), "1")
                .setOutputs("2")
                .build();

        ComputationGraph net1 = new ComputationGraph(conf1);
        net1.init();
        long[] inshape = (rnnDataFormat == NCW)? new long[]{3, 10, 5}: new long[]{3, 5, 10};
        INDArray in = Nd4j.rand(inshape);
        INDArray labels = Nd4j.rand(inshape);

        net1.fit(new DataSet(in, labels));

        byte[] bytes;
        try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
            ModelSerializer.writeModel(net1, baos, true);
            bytes = baos.toByteArray();
        }


        ComputationGraph net2 = ModelSerializer.restoreComputationGraph(new ByteArrayInputStream(bytes), true);


        in = Nd4j.rand(inshape);
        labels = Nd4j.rand(inshape);

        INDArray out1 = net1.outputSingle(in);
        INDArray out2 = net2.outputSingle(in);

        assertEquals(out1, out2);

        net1.setInput(0, in);
        net2.setInput(0, in);
        net1.setLabels(labels);
        net2.setLabels(labels);

        net1.computeGradientAndScore();
        net2.computeGradientAndScore();

        assertEquals(net1.score(), net2.score(), 1e-6);
        assertEquals(net1.gradient().gradient(), net2.gradient().gradient());
    }
}
 

@Test
public void testIssue5472(){
    //https://github.com/deeplearning4j/deeplearning4j/issues/5472

    int in = 2;
    int out = 2;
    ComputationGraphConfiguration.GraphBuilder builder = new NeuralNetConfiguration.Builder()
            .updater(new Adam(0.01))
            .activation(Activation.RELU)
            .graphBuilder()
            .addInputs("IN")
            .setInputTypes(InputType.recurrent(in))
            .addLayer("AUTOENCODER",
                    new VariationalAutoencoder.Builder()
                            .encoderLayerSizes(64)
                            .decoderLayerSizes(64)
                            .nOut(7)
                            .pzxActivationFunction(Activation.IDENTITY)
                            .reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID.getActivationFunction())).build(),
                    "IN")
            .addLayer("RNN", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nOut(128).build()), "AUTOENCODER")
            .addLayer("OUT", new RnnOutputLayer.Builder()
                    .nOut(out)
                    .activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "RNN")
            .setOutputs("OUT")

            ;

    ComputationGraph net = new ComputationGraph(builder.build());
    net.init();

    MultiDataSetIterator iterator = new SingletonMultiDataSetIterator(new MultiDataSet(Nd4j.create(10,in,5), Nd4j.create(10,out,5)));

    EarlyStoppingConfiguration.Builder b = new EarlyStoppingConfiguration.Builder<>()
            .epochTerminationConditions(new MaxEpochsTerminationCondition(10))
            .scoreCalculator(new DataSetLossCalculator(iterator, true))
            .evaluateEveryNEpochs(1)
            .modelSaver(new InMemoryModelSaver<>());

    EarlyStoppingGraphTrainer earlyStoppingGraphTrainer = new EarlyStoppingGraphTrainer(b.build(), net, iterator, null);
    earlyStoppingGraphTrainer.fit();
}
 

@Test
public void testAutoAdditionOfPreprocessors() {
    //FF->RNN and RNN->FF
    MultiLayerConfiguration conf1 =
                    new NeuralNetConfiguration.Builder().list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(5)
                                                    .nOut(6).build())
                                    .layer(1, new GravesLSTM.Builder().nIn(6).nOut(7).build())
                                    .layer(2, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(7)
                                                    .nOut(8).build())
                                    .layer(3, new RnnOutputLayer.Builder().nIn(8).nOut(9).activation(Activation.SOFTMAX).build()).build();
    //Expect preprocessors: layer1: FF->RNN; 2: RNN->FF; 3: FF->RNN
    assertEquals(3, conf1.getInputPreProcessors().size());
    assertTrue(conf1.getInputPreProcess(1) instanceof FeedForwardToRnnPreProcessor);
    assertTrue(conf1.getInputPreProcess(2) instanceof RnnToFeedForwardPreProcessor);
    assertTrue(conf1.getInputPreProcess(3) instanceof FeedForwardToRnnPreProcessor);


    //FF-> CNN, CNN-> FF, FF->RNN
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder().nOut(10)
                                    .kernelSize(5, 5).stride(1, 1).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nOut(6).build())
                    .layer(2, new RnnOutputLayer.Builder().nIn(6).nOut(5).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();
    //Expect preprocessors: 0: FF->CNN; 1: CNN->FF; 2: FF->RNN
    assertEquals(3, conf2.getInputPreProcessors().size());
    assertTrue(conf2.getInputPreProcess(0) instanceof FeedForwardToCnnPreProcessor);
    assertTrue(conf2.getInputPreProcess(1) instanceof CnnToFeedForwardPreProcessor);
    assertTrue(conf2.getInputPreProcess(2) instanceof FeedForwardToRnnPreProcessor);

    //CNN-> FF, FF->RNN - InputType.convolutional instead of convolutionalFlat
    MultiLayerConfiguration conf2a = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder().nOut(10)
                                    .kernelSize(5, 5).stride(1, 1).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nOut(6).build())
                    .layer(2, new RnnOutputLayer.Builder().nIn(6).nOut(5).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutional(28, 28, 1)).build();
    //Expect preprocessors: 1: CNN->FF; 2: FF->RNN
    assertEquals(2, conf2a.getInputPreProcessors().size());
    assertTrue(conf2a.getInputPreProcess(1) instanceof CnnToFeedForwardPreProcessor);
    assertTrue(conf2a.getInputPreProcess(2) instanceof FeedForwardToRnnPreProcessor);


    //FF->CNN and CNN->RNN:
    MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder().nOut(10)
                                    .kernelSize(5, 5).stride(1, 1).build())
                    .layer(1, new GravesLSTM.Builder().nOut(6).build())
                    .layer(2, new RnnOutputLayer.Builder().nIn(6).nOut(5).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();
    //Expect preprocessors: 0: FF->CNN, 1: CNN->RNN;
    assertEquals(2, conf3.getInputPreProcessors().size());
    assertTrue(conf3.getInputPreProcess(0) instanceof FeedForwardToCnnPreProcessor);
    assertTrue(conf3.getInputPreProcess(1) instanceof CnnToRnnPreProcessor);
}
 

@Test
public void testLastTimeStepWithTransfer(){
    int lstmLayerSize = 16;
    int numLabelClasses = 10;
    int numInputs = 5;

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .trainingWorkspaceMode(WorkspaceMode.NONE)
            .inferenceWorkspaceMode(WorkspaceMode.NONE)
            .seed(123)    //Random number generator seed for improved repeatability. Optional.
            .updater(new AdaDelta())
            .weightInit(WeightInit.XAVIER)
            .graphBuilder()
            .addInputs("rr")
            .setInputTypes(InputType.recurrent(30))
            .addLayer("1", new GravesLSTM.Builder().activation(Activation.TANH).nIn(numInputs).nOut(lstmLayerSize).dropOut(0.9).build(), "rr")
            .addLayer("2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(numLabelClasses).build(), "1")

            .setOutputs("2")
            .build();


    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    ComputationGraph updatedModel = new TransferLearning.GraphBuilder(net)
            .addVertex("laststepoutput", new LastTimeStepVertex("rr"), "2")
            .setOutputs("laststepoutput")
            .build();


    INDArray input = Nd4j.rand(new int[]{10, numInputs, 16});

    INDArray[] out = updatedModel.output(input);

    assertNotNull(out);
    assertEquals(1, out.length);
    assertNotNull(out[0]);

    assertArrayEquals(new long[]{10, numLabelClasses}, out[0].shape());

    Map<String,INDArray> acts = updatedModel.feedForward(input, false);

    assertEquals(4, acts.size());   //2 layers + input + vertex output
    assertNotNull(acts.get("laststepoutput"));
    assertArrayEquals(new long[]{10, numLabelClasses}, acts.get("laststepoutput").shape());

    String toString = out[0].toString();
}
 

@Test
public void testOutputMaskingScoreMagnitudes() {
    //Idea: check magnitude of scores, with differing number of values masked out
    //i.e., MSE with zero weight init and 1.0 labels: know what to expect in terms of score

    int nIn = 3;
    int[] timeSeriesLengths = {3, 10};
    int[] outputSizes = {1, 2, 5};
    int[] miniBatchSizes = {1, 4};

    Random r = new Random(12345);

    for (int tsLength : timeSeriesLengths) {
        for (int nOut : outputSizes) {
            for (int miniBatch : miniBatchSizes) {
                for (int nToMask = 0; nToMask < tsLength - 1; nToMask++) {
                    String msg = "tsLen=" + tsLength + ", nOut=" + nOut + ", miniBatch=" + miniBatch;

                    INDArray labelMaskArray = Nd4j.ones(miniBatch, tsLength);
                    for (int i = 0; i < miniBatch; i++) {
                        //For each example: select which outputs to mask...
                        int nMasked = 0;
                        while (nMasked < nToMask) {
                            int tryIdx = r.nextInt(tsLength);
                            if (labelMaskArray.getDouble(i, tryIdx) == 0.0)
                                continue;
                            labelMaskArray.putScalar(new int[] {i, tryIdx}, 0.0);
                            nMasked++;
                        }
                    }

                    INDArray input = Nd4j.rand(new int[] {miniBatch, nIn, tsLength});
                    INDArray labels = Nd4j.ones(miniBatch, nOut, tsLength);

                    ComputationGraphConfiguration conf =
                                    new NeuralNetConfiguration.Builder().seed(12345L)
                                                    .graphBuilder()
                                                    .addInputs("in").addLayer("0",
                                                                    new GravesLSTM.Builder().nIn(nIn).nOut(5)

                                                                                    .dist(new NormalDistribution(0,
                                                                                                    1))
                                                                                    .updater(new NoOp()).build(),
                                                                    "in")
                                                    .addLayer("1", new RnnOutputLayer.Builder(
                                                                    LossFunctions.LossFunction.MSE)
                                                                                    .activation(Activation.IDENTITY)
                                                                                    .nIn(5).nOut(nOut)
                                                                                    .weightInit(WeightInit.ZERO)
                                                                                    .updater(new NoOp()).build(),
                                                                    "0")
                                                    .setOutputs("1").build();
                    ComputationGraph net = new ComputationGraph(conf);
                    net.init();

                    //MSE loss function: 1/n * sum(squaredErrors)... but sum(squaredErrors) = n * (1-0) here -> sum(squaredErrors)
                    double expScore = tsLength - nToMask; //Sum over minibatches, then divide by minibatch size

                    net.setLayerMaskArrays(null, new INDArray[] {labelMaskArray});
                    net.setInput(0, input);
                    net.setLabel(0, labels);

                    net.computeGradientAndScore();
                    double score = net.score();

                    assertEquals(msg, expScore, score, 0.1);
                }
            }
        }
    }
}
 

@Test
public void testOutputMasking() {
    //If labels are masked: want zero outputs for that time step.

    int nIn = 3;
    int[] timeSeriesLengths = {3, 10};
    int[] outputSizes = {1, 2, 5};
    int[] miniBatchSizes = {1, 4};

    Random r = new Random(12345);

    for (int tsLength : timeSeriesLengths) {
        for (int nOut : outputSizes) {
            for (int miniBatch : miniBatchSizes) {
                for (int nToMask = 0; nToMask < tsLength - 1; nToMask++) {
                    INDArray labelMaskArray = Nd4j.ones(miniBatch, tsLength);
                    for (int i = 0; i < miniBatch; i++) {
                        //For each example: select which outputs to mask...
                        int nMasked = 0;
                        while (nMasked < nToMask) {
                            int tryIdx = r.nextInt(tsLength);
                            if (labelMaskArray.getDouble(i, tryIdx) == 0.0)
                                continue;
                            labelMaskArray.putScalar(new int[] {i, tryIdx}, 0.0);
                            nMasked++;
                        }
                    }

                    INDArray input = Nd4j.rand(new int[] {miniBatch, nIn, tsLength});

                    ComputationGraphConfiguration conf =
                                    new NeuralNetConfiguration.Builder().seed(12345L)
                                                    .graphBuilder()
                                                    .addInputs("in").addLayer("0",
                                                                    new GravesLSTM.Builder().nIn(nIn).nOut(5)

                                                                                    .dist(new NormalDistribution(0,
                                                                                                    1))
                                                                                    .updater(new NoOp()).build(),
                                                                    "in")
                                                    .addLayer("1", new RnnOutputLayer.Builder(
                                                                    LossFunctions.LossFunction.MSE)
                                                                                    .activation(Activation.IDENTITY)
                                                                                    .nIn(5).nOut(nOut)
                                                                                    .weightInit(WeightInit.XAVIER)
                                                                                    .updater(new NoOp()).build(),
                                                                    "0")
                                                    .setOutputs("1").build();
                    ComputationGraph net = new ComputationGraph(conf);
                    net.init();

                    ComputationGraphConfiguration conf2 =
                                    new NeuralNetConfiguration.Builder().seed(12345L)
                                                    .graphBuilder()
                                                    .addInputs("in").addLayer("0",
                                                                    new GravesLSTM.Builder().nIn(nIn).nOut(5)

                                                                                    .dist(new NormalDistribution(0,
                                                                                                    1))
                                                                                    .updater(new NoOp()).build(),
                                                                    "in")
                                                    .addLayer("1", new RnnOutputLayer.Builder(
                                                                    LossFunctions.LossFunction.XENT)
                                                                                    .activation(Activation.SIGMOID)
                                                                                    .nIn(5).nOut(nOut)
                                                                                    .weightInit(WeightInit.XAVIER)
                                                                                    .updater(new NoOp()).build(),
                                                                    "0")
                                                    .setOutputs("1").build();
                    ComputationGraph net2 = new ComputationGraph(conf2);
                    net2.init();

                    net.setLayerMaskArrays(null, new INDArray[] {labelMaskArray});
                    net2.setLayerMaskArrays(null, new INDArray[] {labelMaskArray});


                    INDArray out = net.output(input)[0];
                    INDArray out2 = net2.output(input)[0];
                    for (int i = 0; i < miniBatch; i++) {
                        for (int j = 0; j < tsLength; j++) {
                            double m = labelMaskArray.getDouble(i, j);
                            if (m == 0.0) {
                                //Expect outputs to be exactly 0.0
                                INDArray outRow = out.get(NDArrayIndex.point(i), NDArrayIndex.all(),
                                                NDArrayIndex.point(j));
                                INDArray outRow2 = out2.get(NDArrayIndex.point(i), NDArrayIndex.all(),
                                                NDArrayIndex.point(j));
                                for (int k = 0; k < nOut; k++) {
                                    assertEquals(0.0, outRow.getDouble(k), 0.0);
                                    assertEquals(0.0, outRow2.getDouble(k), 0.0);
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
 

@Override
public GravesLSTM getValue(double[] values) {
    GravesLSTM.Builder b = new GravesLSTM.Builder();
    setLayerOptionsBuilder(b, values);
    return b.build();
}
 

protected void setLayerOptionsBuilder(GravesLSTM.Builder builder, double[] values) {
    super.setLayerOptionsBuilder(builder, values);
}