Java Code Examples for org.deeplearning4j.nn.conf.NeuralNetConfiguration#ListBuilder

/** Very simple back-prop config set up for Iris.
 * Learning Rate = 0.1
 * No regularization, no Adagrad, no momentum etc. One iteration.
 */
private static MultiLayerConfiguration getIrisMLPSimpleConfig(int[] hiddenLayerSizes,
                Activation activationFunction) {
    NeuralNetConfiguration.ListBuilder lb = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1))
                .seed(12345L).list();

    for (int i = 0; i < hiddenLayerSizes.length; i++) {
        int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]);
        lb.layer(i, new DenseLayer.Builder().nIn(nIn).nOut(hiddenLayerSizes[i]).weightInit(WeightInit.XAVIER)
                        .activation(activationFunction).build());
    }

    lb.layer(hiddenLayerSizes.length,
                    new OutputLayer.Builder(LossFunction.MCXENT).nIn(hiddenLayerSizes[hiddenLayerSizes.length - 1])
                                    .nOut(3).weightInit(WeightInit.XAVIER)
                                    .activation(activationFunction.equals(Activation.IDENTITY) ? Activation.IDENTITY
                                                    : Activation.SOFTMAX)
                                    .build());

    return lb.build();
}
 

@Test
public void testLRN() throws Exception {
    List<String> labels = new ArrayList<>(Arrays.asList("Zico", "Ziwang_Xu"));
    File dir = testDir.newFolder();
    new ClassPathResource("lfwtest/").copyDirectory(dir);
    String rootDir = dir.getAbsolutePath();

    RecordReader reader = new ImageRecordReader(28, 28, 3);
    reader.initialize(new FileSplit(new File(rootDir)));
    DataSetIterator recordReader = new RecordReaderDataSetIterator(reader, 10, 1, labels.size());
    labels.remove("lfwtest");
    NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLRN();
    builder.setInputType(InputType.convolutional(28, 28, 3));

    MultiLayerConfiguration conf = builder.build();

    ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(3).getLayer();
    assertEquals(6, layer2.getNIn());

}
 

@Test
public void testMultiChannel() throws Exception {
    INDArray in = Nd4j.rand(new int[] {10, 3, 28, 28});
    INDArray labels = Nd4j.rand(10, 2);
    DataSet next = new DataSet(in, labels);

    NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLFW();
    builder.setInputType(InputType.convolutional(28, 28, 3));
    MultiLayerConfiguration conf = builder.build();
    ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(2).getLayer();
    assertEquals(6, layer2.getNIn());

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.fit(next);
}
 

private MultiLayerNetwork getCnnLossNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm){
    NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .convolutionMode(cm)
            .list()
            .layer(new ConvolutionLayer.Builder()
                    .kernelSize(3, 3)
                    .stride(2, 2)
                    .activation(Activation.TANH)
                    .dataFormat(format)
                    .nOut(3)
                    .helperAllowFallback(false)
                    .build());
    if(setOnLayerAlso){
        builder.layer(new CnnLossLayer.Builder().format(format).activation(Activation.SOFTMAX).build());
    } else {
        builder.layer(new CnnLossLayer.Builder().activation(Activation.SOFTMAX).build());
    }

    builder.setInputType(InputType.convolutional(12, 12, 3, format));

    MultiLayerNetwork net = new MultiLayerNetwork(builder.build());
    net.init();
    return net;
}
 

@Override
protected MultiLayerConfiguration getNetworkConfiguration() {
    NeuralNetConfiguration.ListBuilder factory = new NeuralNetConfiguration.Builder().seed(6).updater(new Nesterovs(learnRatio, momentum)).weightInit(WeightInit.XAVIER_UNIFORM).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(weightRegularization).list();
    factory.layer(0, new DenseLayer.Builder().nIn(inputDimension).nOut(hiddenDimension).activation(Activation.fromString(hiddenActivation)).build());
    factory.layer(1, new OutputLayer.Builder(new AutoRecLearner(maskData)).nIn(hiddenDimension).nOut(inputDimension).activation(Activation.fromString(outputActivation)).build());
    MultiLayerConfiguration configuration = factory.pretrain(false).backprop(true).build();
    return configuration;
}
 

@Override
protected MultiLayerConfiguration getNetworkConfiguration() {
    NeuralNetConfiguration.ListBuilder factory = new NeuralNetConfiguration.Builder().seed(6)
            // .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue)
            // .gradientNormalizationThreshold(1.0)
            .updater(new Nesterovs(learnRatio, momentum)).weightInit(WeightInit.XAVIER).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(weightRegularization).list();
    factory.layer(0, new CDAEConfiguration.Builder().nIn(inputDimension).nOut(hiddenDimension).activation(Activation.fromString(hiddenActivation)).setNumUsers(userSize).build());
    factory.layer(1, new OutputLayer.Builder().nIn(hiddenDimension).nOut(inputDimension).lossFunction(LossFunctions.LossFunction.SQUARED_LOSS).activation(Activation.fromString(outputActivation)).build());
    factory.pretrain(false).backprop(true);
    MultiLayerConfiguration configuration = factory.build();
    return configuration;
}
 

private MultiLayerNetwork getNetWithLayer(Layer layer, CNN2DFormat format, ConvolutionMode cm, InputType inputType) {
    NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder()
            .dataType(this.dataType)
            .seed(12345)
            .convolutionMode(cm)
            .list()
            .layer(new ConvolutionLayer.Builder()
                    .kernelSize(3, 3)
                    .stride(2, 2)
                    .activation(Activation.TANH)
                    .dataFormat(format)
                    .nOut(3)
                    .helperAllowFallback(false)
                    .build())
            .layer(layer)
            .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).build())
            .setInputType(inputType != null ? inputType : InputType.convolutional(12, 12, 3, format));

    if(format == CNN2DFormat.NHWC && !(layer instanceof GlobalPoolingLayer)){
        //Add a preprocessor due to the differences in how NHWC and NCHW activations are flattened
        //DL4J's flattening behaviour matches Keras (hence TF) for import compatibility
        builder.inputPreProcessor(2, new ComposableInputPreProcessor(new NHWCToNCHWPreprocessor(), new CnnToFeedForwardPreProcessor()));
    }

    MultiLayerNetwork net = new MultiLayerNetwork(builder.build());
    net.init();
    return net;
}
 

public DQN buildDQN(int shapeInputs[], int numOutputs) {

        if (shapeInputs.length == 1)
            throw new AssertionError("Impossible to apply convolutional layer on a shape == 1");


        NeuralNetConfiguration.ListBuilder confB = new NeuralNetConfiguration.Builder().seed(Constants.NEURAL_NET_SEED)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .l2(conf.getL2())
                        .updater(conf.getUpdater() != null ? conf.getUpdater() : new Adam())
                        .weightInit(WeightInit.XAVIER).l2(conf.getL2()).list()
                        .layer(0, new ConvolutionLayer.Builder(8, 8).nIn(shapeInputs[0]).nOut(16).stride(4, 4)
                                        .activation(Activation.RELU).build());


        confB.layer(1, new ConvolutionLayer.Builder(4, 4).nOut(32).stride(2, 2).activation(Activation.RELU).build());

        confB.layer(2, new DenseLayer.Builder().nOut(256).activation(Activation.RELU).build());

        confB.layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nOut(numOutputs)
                        .build());

        confB.setInputType(InputType.convolutional(shapeInputs[1], shapeInputs[2], shapeInputs[0]));
        MultiLayerConfiguration mlnconf = confB.build();
        MultiLayerNetwork model = new MultiLayerNetwork(mlnconf);
        model.init();
        if (conf.getListeners() != null) {
            model.setListeners(conf.getListeners());
        } else {
            model.setListeners(new ScoreIterationListener(Constants.NEURAL_NET_ITERATION_LISTENER));
        }

        return new DQN(model);
    }
 

private MultiLayerNetwork getNetWithLayer(Layer layer, RNNFormat format, boolean lastTimeStep, boolean maskZeros) {
    if (maskZeros){
        layer = new MaskZeroLayer.Builder().setMaskValue(0.).setUnderlying(layer).build();
    }
    if(lastTimeStep){
        layer = new LastTimeStep(layer);
    }
    NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .list()
            .layer(new LSTM.Builder()
                    .nIn(3)
                    .activation(Activation.TANH)
                    .dataFormat(format)
                    .nOut(3)
                    .helperAllowFallback(false)
                    .build())
            .layer(layer)
            .layer(
                    (lastTimeStep)?new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).build():
    new RnnOutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).dataFormat(format).build()
            )
            .setInputType(InputType.recurrent(3, 12, format));

    MultiLayerNetwork net = new MultiLayerNetwork(builder.build());
    net.init();
    return net;
}
 

@Test
    public void testSeparableConv2D() {
        int nOut = 2;

        int[] minibatchSizes = new int[]{1, 3};
        int width = 6;
        int height = 6;
        int inputDepth = 3;

        Nd4j.getRandom().setSeed(12345);

        int[] ks = new int[]{1, 3, 3, 1, 3};
        int[] ss = new int[]{1, 1, 1, 2, 2};
        int[] ds = new int[]{1, 1, 2, 2, 2};
        ConvolutionMode[] cms = new ConvolutionMode[]{Truncate, Truncate, Truncate, Truncate, Truncate};
        int[] mb = new int[]{1, 1, 1, 3, 3};

        for (int t = 0; t < ks.length; t++) {

            int k = ks[t];
            int s = ss[t];
            int d = ds[t];
            ConvolutionMode cm = cms[t];
            int minibatchSize = mb[t];

            //Use larger input with larger dilation values (to avoid invalid config)
            int w = d * width;
            int h = d * height;

            INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth);
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(new int[]{i, i % nOut}, 1.0);
            }

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                    .dataType(DataType.DOUBLE)
                    .updater(new NoOp())
                    .activation(Activation.TANH)
                    .convolutionMode(cm)
                    .list()
                    .layer(new SeparableConvolution2D.Builder().name("Separable conv 2D layer")
                            .kernelSize(k, k)
                            .stride(s, s)
                            .dilation(d, d)
                            .depthMultiplier(3)
                            .nIn(inputDepth).nOut(2).build());

            MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

//            for (int i = 0; i < net.getLayers().length; i++) {
//                System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
//            }

            String msg = " - mb=" + minibatchSize + ", k="
                    + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(
                    new GradientCheckUtil.MLNConfig().net(net)
                            .input(input).labels(labels)
                            .subset(true).maxPerParam(50));

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }
 

@Test
public void testMismatchedInputLabelLength(){

    for( int i=0; i<2; i++ ){

        NeuralNetConfiguration.ListBuilder lb = new NeuralNetConfiguration.Builder()

                .list()
                .layer(new SimpleRnn.Builder().nIn(5).nOut(5).dataFormat(rnnDataFormat).build());

        switch (i){
            case 0:
                lb.layer(new RnnOutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(5).dataFormat(rnnDataFormat).build());
                break;
            case 1:
                lb.layer(new RnnLossLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).dataFormat(rnnDataFormat).build());
                break;
            default:
                throw new RuntimeException();
        }

        MultiLayerConfiguration conf = lb.build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray in = Nd4j.rand(DataType.FLOAT, 3, 5, 5);
        INDArray l = TestUtils.randomOneHotTimeSeries(rnnDataFormat, 3, 5, 10, new Random(12345));
        try{
            net.fit(in,l);
        } catch (Throwable t){
            String msg = t.getMessage();
            if(msg == null)
                t.printStackTrace();
            System.out.println(i);
            assertTrue(msg, msg != null && msg.contains("sequence length") && msg.contains("input") && msg.contains("label"));
        }

    }


}
 

@Test
public void testDepthwiseConv2D() {
    int nIn = 3;
    int depthMultiplier = 2;
    int nOut = nIn * depthMultiplier;

    int width = 5;
    int height = 5;

    Nd4j.getRandom().setSeed(12345);

    int[] ks = new int[]{1,3,3,1,3};
    int[] ss = new int[]{1,1,1,2,2};
    ConvolutionMode[] cms = new ConvolutionMode[]{
            Truncate, Truncate, Truncate, Truncate, Truncate};
    int[] mb = new int[]{1,1,1,3,3};

    boolean nchw = format == CNN2DFormat.NCHW;

    for( int t=0; t<ks.length; t++ ){

        int k = ks[t];
        int s = ss[t];
        ConvolutionMode cm = cms[t];
        int minibatchSize = mb[t];

        long[] inShape = nchw ? new long[]{minibatchSize, nIn, height, width} : new long[]{minibatchSize, height, width, nIn};
        INDArray input = Nd4j.rand(DataType.DOUBLE, inShape);
        INDArray labels = Nd4j.zeros(minibatchSize, nOut);
        for (int i = 0; i < minibatchSize; i++) {
            labels.putScalar(new int[]{i, i % nOut}, 1.0);
        }

        NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                .dataType(DataType.DOUBLE)
                .updater(new NoOp())
                .activation(Activation.TANH)
                .convolutionMode(cm)
                .list()
                .layer(new Convolution2D.Builder().kernelSize(1, 1).stride(1, 1).nIn(nIn).nOut(nIn).build())
                .layer(new DepthwiseConvolution2D.Builder().name("depth-wise conv 2D layer")
                        .cudnnAllowFallback(false)
                        .kernelSize(k, k)
                        .stride(s, s)
                        .depthMultiplier(depthMultiplier)
                        .nIn(nIn).build()); // nOut = nIn * depthMultiplier

        MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                .activation(Activation.SOFTMAX).nOut(nOut).build())
                .setInputType(InputType.convolutional(height, width, nIn, format)).build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        for (int i = 0; i < net.getLayers().length; i++) {
            System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
        }

        String msg = " - mb=" + minibatchSize + ", k="
                + k + ", nIn=" + nIn + ", depthMul=" + depthMultiplier + ", s=" + s + ", cm=" + cm;
        System.out.println(msg);

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
                .labels(labels).subset(true).maxPerParam(256));

        assertTrue(msg, gradOK);

        TestUtils.testModelSerialization(net);
    }
}
 

@Test
public void testCnnDilated() {
    int nOut = 2;
    int minibatchSize = 2;
    int width = 8;
    int height = 8;
    int inputDepth = 2;

    Nd4j.getRandom().setSeed(12345);

    boolean[] sub = new boolean[]{true, true, false, true, false};
    int[] stride = new int[]{1, 1, 1, 2, 2};
    int[] kernel = new int[]{2, 3, 3, 3, 3};
    int[] ds = new int[]{2, 2, 3, 3, 2};
    ConvolutionMode[] cms = new ConvolutionMode[]{Same, Truncate, Truncate, Same, Truncate};

    boolean nchw = format == CNN2DFormat.NCHW;

    for (int t = 0; t < sub.length; t++) {
        boolean subsampling = sub[t];
        int s = stride[t];
        int k = kernel[t];
        int d = ds[t];
        ConvolutionMode cm = cms[t];

        //Use larger input with larger dilation values (to avoid invalid config)
        int w = d * width;
        int h = d * height;

        long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, h, w} : new long[]{minibatchSize, h, w, inputDepth};
        INDArray input = Nd4j.rand(DataType.DOUBLE, inShape);
        INDArray labels = Nd4j.zeros(minibatchSize, nOut);
        for (int i = 0; i < minibatchSize; i++) {
            labels.putScalar(new int[]{i, i % nOut}, 1.0);
        }

        NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                .dataType(DataType.DOUBLE)
                .updater(new NoOp())
                .activation(Activation.TANH).convolutionMode(cm).list()
                .layer(new ConvolutionLayer.Builder().name("layer 0")
                        .kernelSize(k, k)
                        .stride(s, s)
                        .dilation(d, d)
                        .nIn(inputDepth).nOut(2).build());
        if (subsampling) {
            b.layer(new SubsamplingLayer.Builder()
                    .poolingType(SubsamplingLayer.PoolingType.MAX)
                    .kernelSize(k, k)
                    .stride(s, s)
                    .dilation(d, d)
                    .build());
        } else {
            b.layer(new ConvolutionLayer.Builder().nIn(2).nOut(2)
                    .kernelSize(k, k)
                    .stride(s, s)
                    .dilation(d, d)
                    .build());
        }

        MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                .activation(Activation.SOFTMAX).nOut(nOut).build())
                .setInputType(InputType.convolutional(h, w, inputDepth, format)).build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        for (int i = 0; i < net.getLayers().length; i++) {
            System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
        }

        String msg = (subsampling ? "subsampling" : "conv") + " - mb=" + minibatchSize + ", k="
                + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
        System.out.println(msg);

        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        assertTrue(msg, gradOK);

        TestUtils.testModelSerialization(net);
    }
}
 

@Test
public void testSeparableConv2D() {
    int nOut = 2;
    int width = 6;
    int height = 6;
    int inputDepth = 3;

    Nd4j.getRandom().setSeed(12345);

    int[] ks = new int[]{1, 3, 3, 1, 3};
    int[] ss = new int[]{1, 1, 1, 2, 2};
    int[] ds = new int[]{1, 1, 2, 2, 2};
    ConvolutionMode[] cms = new ConvolutionMode[]{Truncate, Truncate, Truncate, Truncate, Truncate};
    int[] mb = new int[]{1, 1, 1, 3, 3};

    boolean nchw = format == CNN2DFormat.NCHW;

    for (int t = 0; t < ks.length; t++) {

        int k = ks[t];
        int s = ss[t];
        int d = ds[t];
        ConvolutionMode cm = cms[t];
        int minibatchSize = mb[t];

        //Use larger input with larger dilation values (to avoid invalid config)
        int w = d * width;
        int h = d * height;

        long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, h, w} : new long[]{minibatchSize, h, w, inputDepth};
        INDArray input = Nd4j.rand(DataType.DOUBLE, inShape);
        INDArray labels = Nd4j.zeros(minibatchSize, nOut);
        for (int i = 0; i < minibatchSize; i++) {
            labels.putScalar(new int[]{i, i % nOut}, 1.0);
        }

        NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                .dataType(DataType.DOUBLE)
                .updater(new NoOp())
                .activation(Activation.TANH)
                .convolutionMode(cm)
                .list()
                .layer(new SeparableConvolution2D.Builder().name("Separable conv 2D layer")
                        .kernelSize(k, k)
                        .stride(s, s)
                        .dilation(d, d)
                        .depthMultiplier(3)
                        .nIn(inputDepth).nOut(2).build());

        MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                .activation(Activation.SOFTMAX).nOut(nOut).build())
                .setInputType(InputType.convolutional(h, w, inputDepth, format)).build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        for (int i = 0; i < net.getLayers().length; i++) {
            System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
        }

        String msg = " - mb=" + minibatchSize + ", k="
                + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
        System.out.println(msg);

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
                .labels(labels).subset(true).maxPerParam(50));  //Most params are in output layer

        assertTrue(msg, gradOK);

        TestUtils.testModelSerialization(net);
    }
}
 

@Override
public DL4JConfiguration getValue(double[] values) {
    //First: create layer configs
    List<org.deeplearning4j.nn.conf.layers.Layer> layers = new ArrayList<>();
    for (LayerConf c : layerSpaces) {
        int n = c.numLayers.getValue(values);
        if (c.duplicateConfig) {
            //Generate N identical configs
            org.deeplearning4j.nn.conf.layers.Layer l = c.layerSpace.getValue(values);
            for (int i = 0; i < n; i++) {
                layers.add(l.clone());
            }
        } else {
            throw new UnsupportedOperationException("Not yet implemented");
        }
    }

    //Create MultiLayerConfiguration...
    NeuralNetConfiguration.Builder builder = randomGlobalConf(values);

    NeuralNetConfiguration.ListBuilder listBuilder = builder.list();
    for (int i = 0; i < layers.size(); i++) {
        listBuilder.layer(i, layers.get(i));
    }

    if (backpropType != null)
        listBuilder.backpropType(backpropType.getValue(values));
    if (tbpttFwdLength != null)
        listBuilder.tBPTTForwardLength(tbpttFwdLength.getValue(values));
    if (tbpttBwdLength != null)
        listBuilder.tBPTTBackwardLength(tbpttBwdLength.getValue(values));
    if (inputType != null)
        listBuilder.setInputType(inputType.getValue(values));
    if (inputPreProcessors != null)
        listBuilder.setInputPreProcessors(inputPreProcessors.getValue(values));
    listBuilder.validateOutputLayerConfig(validateOutputLayerConfig);

    MultiLayerConfiguration configuration = listBuilder.build();

    if (trainingWorkspaceMode != null)
        configuration.setTrainingWorkspaceMode(trainingWorkspaceMode);
    if (inferenceWorkspaceMode != null)
        configuration.setInferenceWorkspaceMode(inferenceWorkspaceMode);


    return new DL4JConfiguration(configuration, earlyStoppingConfiguration, numEpochs);
}
 

@Test
    public void testCnnDilated() {
        int nOut = 2;

        int minibatchSize = 2;
        int width = 8;
        int height = 8;
        int inputDepth = 2;

        Nd4j.getRandom().setSeed(12345);

        boolean[] sub = new boolean[]{true, true, false, true, false};
        int[] stride = new int[]{1, 1, 1, 2, 2};
        int[] kernel = new int[]{2, 3, 3, 3, 3};
        int[] ds = new int[]{2, 2, 3, 3, 2};
        ConvolutionMode[] cms = new ConvolutionMode[]{Same, Truncate, Truncate, Same, Truncate};


        for (int t = 0; t < sub.length; t++) {

            boolean subsampling = sub[t];
            int s = stride[t];
            int k = kernel[t];
            int d = ds[t];
            ConvolutionMode cm = cms[t];

            //Use larger input with larger dilation values (to avoid invalid config)
            int w = d * width;
            int h = d * height;

            INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth);
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(new int[]{i, i % nOut}, 1.0);
            }

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                    .dataType(DataType.DOUBLE)
                    .updater(new NoOp())
                    .activation(Activation.TANH).convolutionMode(cm).list()
                    .layer(new ConvolutionLayer.Builder().name("layer 0")
                            .kernelSize(k, k)
                            .stride(s, s)
                            .dilation(d, d)
                            .nIn(inputDepth).nOut(2).build());
            if (subsampling) {
                b.layer(new SubsamplingLayer.Builder()
                        .poolingType(SubsamplingLayer.PoolingType.MAX)
                        .kernelSize(k, k)
                        .stride(s, s)
                        .dilation(d, d)
                        .build());
            } else {
                b.layer(new ConvolutionLayer.Builder().nIn(2).nOut(2)
                        .kernelSize(k, k)
                        .stride(s, s)
                        .dilation(d, d)
                        .build());
            }

            MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

//            for (int i = 0; i < net.getLayers().length; i++) {
//                System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
//            }

            String msg = (subsampling ? "subsampling" : "conv") + " - mb=" + minibatchSize + ", k="
                    + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }
 

@Test
    public void testGradientLSTMFull() {

        int timeSeriesLength = 4;
        int nIn = 3;
        int layerSize = 4;
        int nOut = 2;
        int miniBatchSize = 2;

        boolean[] gravesLSTM = new boolean[] {true, false};

        for (boolean graves : gravesLSTM) {

            Random r = new Random(12345L);
            INDArray input = Nd4j.rand(new int[]{miniBatchSize, nIn, timeSeriesLength}, 'f').subi(0.5);

            INDArray labels = Nd4j.zeros(miniBatchSize, nOut, timeSeriesLength);
            for (int i = 0; i < miniBatchSize; i++) {
                for (int j = 0; j < timeSeriesLength; j++) {
                    int idx = r.nextInt(nOut);
                    labels.putScalar(new int[] {i, idx, j}, 1.0f);
                }
            }


            //use l2vals[i] with l1vals[i]
            double[] l2vals = {0.4, 0.0};
            double[] l1vals = {0.0, 0.5};
            double[] biasL2 = {0.3, 0.0};
            double[] biasL1 = {0.0, 0.6};
            Activation[] activFns = {Activation.TANH, Activation.SOFTSIGN};
            LossFunction[] lossFunctions = {LossFunction.MCXENT, LossFunction.MSE};
            Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH};

            for (int i = 0; i < l2vals.length; i++) {

                LossFunction lf = lossFunctions[i];
                Activation outputActivation = outputActivations[i];
                double l2 = l2vals[i];
                double l1 = l1vals[i];
                Activation afn = activFns[i];

                NeuralNetConfiguration.Builder conf =
                        new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .seed(12345L)
                                .dist(new NormalDistribution(0, 1)).updater(new NoOp());

                if (l1 > 0.0)
                    conf.l1(l1);
                if (l2 > 0.0)
                    conf.l2(l2);
                if (biasL2[i] > 0)
                    conf.l2Bias(biasL2[i]);
                if (biasL1[i] > 0)
                    conf.l1Bias(biasL1[i]);

                Layer layer;
                if (graves) {
                    layer = new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).activation(afn).build();
                } else {
                    layer = new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(afn).build();
                }

                NeuralNetConfiguration.ListBuilder conf2 = conf.list().layer(0, layer)
                        .layer(1, new RnnOutputLayer.Builder(lf).activation(outputActivation)
                                .nIn(layerSize).nOut(nOut).build())
                        ;

                MultiLayerNetwork mln = new MultiLayerNetwork(conf2.build());
                mln.init();

                String testName = "testGradientLSTMFull(" + (graves ? "GravesLSTM" : "LSTM")
                        + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation="
                        + outputActivation + ", l2=" + l2 + ", l1=" + l1;
                if (PRINT_RESULTS) {
                    System.out.println(testName);
//                    for (int j = 0; j < mln.getnLayers(); j++)
//                        System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input)
                        .labels(labels).subset(true).maxPerParam(128));

                assertTrue(testName, gradOK);
                TestUtils.testModelSerialization(mln);
            }
        }
    }
 

/**
 * Configure a MultiLayerConfiguration from this Keras Sequential model configuration.
 *
 * @return MultiLayerConfiguration
 */
public MultiLayerConfiguration getMultiLayerConfiguration()
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    if (!this.className.equals(config.getFieldClassNameSequential()))
        throw new InvalidKerasConfigurationException(
                "Keras model class name " + this.className + " incompatible with MultiLayerNetwork");
    if (this.inputLayerNames.size() != 1)
        throw new InvalidKerasConfigurationException(
                "MultiLayerNetwork expects only 1 input (found " + this.inputLayerNames.size() + ")");
    if (this.outputLayerNames.size() != 1)
        throw new InvalidKerasConfigurationException(
                "MultiLayerNetwork expects only 1 output (found " + this.outputLayerNames.size() + ")");

    NeuralNetConfiguration.Builder modelBuilder = new NeuralNetConfiguration.Builder();

    if (optimizer != null) {
        modelBuilder.updater(optimizer);
    }

    NeuralNetConfiguration.ListBuilder listBuilder = modelBuilder.list();

    /* Add layers one at a time. */
    KerasLayer prevLayer = null;
    int layerIndex = 0;
    for (KerasLayer layer : this.layersOrdered) {
        if (layer.isLayer()) {
            int nbInbound = layer.getInboundLayerNames().size();
            if (nbInbound != 1)
                throw new InvalidKerasConfigurationException(
                        "Layers in MultiLayerConfiguration must have exactly one inbound layer (found "
                                + nbInbound + " for layer " + layer.getLayerName() + ")");
            if (prevLayer != null) {
                InputType[] inputTypes = new InputType[1];
                InputPreProcessor preprocessor;
                if (prevLayer.isInputPreProcessor()) {
                    inputTypes[0] = this.outputTypes.get(prevLayer.getInboundLayerNames().get(0));
                    preprocessor = prevLayer.getInputPreprocessor(inputTypes);
                } else {
                    inputTypes[0] = this.outputTypes.get(prevLayer.getLayerName());
                    preprocessor = layer.getInputPreprocessor(inputTypes);
                }
                if (preprocessor != null)
                    listBuilder.inputPreProcessor(layerIndex, preprocessor);
            }
            listBuilder.layer(layerIndex++, layer.getLayer());
        } else if (layer.getVertex() != null)
            throw new InvalidKerasConfigurationException("Cannot add vertex to MultiLayerConfiguration (class name "
                    + layer.getClassName() + ", layer name " + layer.getLayerName() + ")");
        prevLayer = layer;
    }

    InputType inputType = this.layersOrdered.get(0).getOutputType();
    if (inputType != null)
        listBuilder.setInputType(inputType);

    /* Whether to use standard backprop (or BPTT) or truncated BPTT. */
    if (this.useTruncatedBPTT && this.truncatedBPTT > 0)
        listBuilder.backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(truncatedBPTT)
                .tBPTTBackwardLength(truncatedBPTT);
    else
        listBuilder.backpropType(BackpropType.Standard);
    return listBuilder.build();
}
 

public ActorCriticSeparate buildActorCritic(int[] numInputs, int numOutputs) {
    int nIn = 1;
    for (int i : numInputs) {
        nIn *= i;
    }
    NeuralNetConfiguration.ListBuilder confB = new NeuralNetConfiguration.Builder().seed(Constants.NEURAL_NET_SEED)
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(conf.getUpdater() != null ? conf.getUpdater() : new Adam())
            .weightInit(WeightInit.XAVIER)
            .l2(conf.getL2())
            .list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(conf.getNumHiddenNodes())
                    .activation(Activation.RELU).build());


    for (int i = 1; i < conf.getNumLayers(); i++) {
        confB.layer(i, new DenseLayer.Builder().nIn(conf.getNumHiddenNodes()).nOut(conf.getNumHiddenNodes())
                .activation(Activation.RELU).build());
    }

    if (conf.isUseLSTM()) {
        confB.layer(conf.getNumLayers(), new LSTM.Builder().nOut(conf.getNumHiddenNodes()).activation(Activation.TANH).build());

        confB.layer(conf.getNumLayers() + 1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY)
                .nIn(conf.getNumHiddenNodes()).nOut(1).build());
    } else {
        confB.layer(conf.getNumLayers(), new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY)
                .nIn(conf.getNumHiddenNodes()).nOut(1).build());
    }

    confB.setInputType(conf.isUseLSTM() ? InputType.recurrent(nIn) : InputType.feedForward(nIn));
    MultiLayerConfiguration mlnconf2 = confB.build();
    MultiLayerNetwork model = new MultiLayerNetwork(mlnconf2);
    model.init();
    if (conf.getListeners() != null) {
        model.setListeners(conf.getListeners());
    } else {
        model.setListeners(new ScoreIterationListener(Constants.NEURAL_NET_ITERATION_LISTENER));
    }

    NeuralNetConfiguration.ListBuilder confB2 = new NeuralNetConfiguration.Builder().seed(Constants.NEURAL_NET_SEED)
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(conf.getUpdater() != null ? conf.getUpdater() : new Adam())
            .weightInit(WeightInit.XAVIER)
            //.regularization(true)
            //.l2(conf.getL2())
            .list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(conf.getNumHiddenNodes())
                    .activation(Activation.RELU).build());


    for (int i = 1; i < conf.getNumLayers(); i++) {
        confB2.layer(i, new DenseLayer.Builder().nIn(conf.getNumHiddenNodes()).nOut(conf.getNumHiddenNodes())
                .activation(Activation.RELU).build());
    }

    if (conf.isUseLSTM()) {
        confB2.layer(conf.getNumLayers(), new LSTM.Builder().nOut(conf.getNumHiddenNodes()).activation(Activation.TANH).build());

        confB2.layer(conf.getNumLayers() + 1, new RnnOutputLayer.Builder(new ActorCriticLoss())
                .activation(Activation.SOFTMAX).nIn(conf.getNumHiddenNodes()).nOut(numOutputs).build());
    } else {
        confB2.layer(conf.getNumLayers(), new OutputLayer.Builder(new ActorCriticLoss())
                .activation(Activation.SOFTMAX).nIn(conf.getNumHiddenNodes()).nOut(numOutputs).build());
    }

    confB2.setInputType(conf.isUseLSTM() ? InputType.recurrent(nIn) : InputType.feedForward(nIn));
    MultiLayerConfiguration mlnconf = confB2.build();
    MultiLayerNetwork model2 = new MultiLayerNetwork(mlnconf);
    model2.init();
    if (conf.getListeners() != null) {
        model2.setListeners(conf.getListeners());
    } else {
        model2.setListeners(new ScoreIterationListener(Constants.NEURAL_NET_ITERATION_LISTENER));
    }


    return new ActorCriticSeparate(model, model2);
}
 

@Test
    public void testDepthwiseConv2D() {
        int nIn = 3;
        int depthMultiplier = 2;
        int nOut = nIn * depthMultiplier;

        int width = 5;
        int height = 5;

        Nd4j.getRandom().setSeed(12345);

        int[] ks = new int[]{1,3,3,1,3};
        int[] ss = new int[]{1,1,1,2,2};
        ConvolutionMode[] cms = new ConvolutionMode[]{
                Truncate, Truncate, Truncate, Truncate, Truncate};
        int[] mb = new int[]{1,1,1,3,3};

        for( int t=0; t<ks.length; t++ ){

            int k = ks[t];
            int s = ss[t];
            ConvolutionMode cm = cms[t];
            int minibatchSize = mb[t];


            INDArray input = Nd4j.rand(minibatchSize, width * height * nIn);
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(new int[]{i, i % nOut}, 1.0);
            }

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                    .dataType(DataType.DOUBLE)
                    .updater(new NoOp())
                    .activation(Activation.TANH)
                    .convolutionMode(cm)
                    .list()
                    .layer(new Convolution2D.Builder().kernelSize(1, 1).stride(1, 1).nIn(nIn).nOut(nIn).build())
                    .layer(new DepthwiseConvolution2D.Builder().name("depth-wise conv 2D layer")
                            .cudnnAllowFallback(false)
                            .kernelSize(k, k)
                            .stride(s, s)
                            .depthMultiplier(depthMultiplier)
                            .nIn(nIn).build()); // nOut = nIn * depthMultiplier

            MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutionalFlat(height, width, nIn)).build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

//            for (int i = 0; i < net.getLayers().length; i++) {
//                System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
//            }

            String msg = " - mb=" + minibatchSize + ", k="
                    + k + ", nIn=" + nIn + ", depthMul=" + depthMultiplier + ", s=" + s + ", cm=" + cm;
            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(
                    new GradientCheckUtil.MLNConfig().net(net)
                            .input(input).labels(labels)
                            .subset(true).maxPerParam(256));

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }