Java Code Examples for org.deeplearning4j.nn.conf.MultiLayerConfiguration#fromJson()

@Override
public VariationalAutoencoder getVaeLayer() {
    MultiLayerNetwork network =
                    new MultiLayerNetwork(MultiLayerConfiguration.fromJson((String) jsonConfig.getValue()));
    network.init();
    INDArray val = ((INDArray) params.value()).unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException(
                        "Network did not have same number of parameters as the broadcast set parameters");
    network.setParameters(val);

    Layer l = network.getLayer(0);
    if (!(l instanceof VariationalAutoencoder)) {
        throw new RuntimeException(
                        "Cannot use VaeReconstructionErrorWithKeyFunction on network that doesn't have a VAE "
                                        + "layer as layer 0. Layer type: " + l.getClass());
    }
    return (VariationalAutoencoder) l;
}
 

/**
 * Load the model from the given file path
 * @param path the path of the file to "guess"
 *
 * @return the loaded model
 * @throws Exception
 */
public static Object loadConfigGuess(String path) throws Exception {
    String input = FileUtils.readFileToString(new File(path));
    //note here that we load json BEFORE YAML. YAML
    //turns out to load just fine *accidentally*
    try {
        return MultiLayerConfiguration.fromJson(input);
    } catch (Exception e) {
        log.warn("Tried multi layer config from json", e);
        try {
            return KerasModelImport.importKerasModelConfiguration(path);
        } catch (Exception e1) {
            log.warn("Tried keras model config", e);
            try {
                return KerasModelImport.importKerasSequentialConfiguration(path);
            } catch (Exception e2) {
                log.warn("Tried keras sequence config", e);
                try {
                    return ComputationGraphConfiguration.fromJson(input);
                } catch (Exception e3) {
                    log.warn("Tried computation graph from json");
                    try {
                        return MultiLayerConfiguration.fromYaml(input);
                    } catch (Exception e4) {
                        log.warn("Tried multi layer configuration from yaml");
                        try {
                            return ComputationGraphConfiguration.fromYaml(input);
                        } catch (Exception e5) {
                            throw new ModelGuesserException("Unable to load configuration from path " + path
                                    + " (invalid config file or not a known config type)");
                        }
                    }
                }
            }
        }
    }
}
 

@Test
public void testFrozenNewFormat(){
    MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder()
            .list()
            .layer(0, new FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).build()))
            .build();

    String json = configuration.toJson();
    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json);
    assertEquals(configuration, fromJson);
}
 

@Override
public Iterator<Tuple2<Integer, Double>> call(Iterator<DataSet> dataSetIterator) throws Exception {
    if (!dataSetIterator.hasNext()) {
        return Collections.singletonList(new Tuple2<>(0, 0.0)).iterator();
    }

    DataSetIterator iter = new IteratorDataSetIterator(dataSetIterator, minibatchSize); //Does batching where appropriate

    MultiLayerNetwork network = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(json));
    network.init();
    INDArray val = params.value().unsafeDuplication(); //.value() object will be shared by all executors on each machine -> OK, as params are not modified by score function
    if (val.length() != network.numParams(false))
        throw new IllegalStateException(
                        "Network did not have same number of parameters as the broadcast set parameters");
    network.setParameters(val);

    List<Tuple2<Integer, Double>> out = new ArrayList<>();
    while (iter.hasNext()) {
        DataSet ds = iter.next();
        double score = network.score(ds, false);

        val numExamples = (int) ds.getFeatures().size(0);
        out.add(new Tuple2<>(numExamples, score * numExamples));
    }

    Nd4j.getExecutioner().commit();

    return out.iterator();
}
 

@Test
public void testSerialization() {

    final MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new AdaGrad(0.1))
                    .l2(0.001)
                    .seed(12345).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder()
                                    .activation(Activation.TANH).nIn(2).nOut(2)
                                    .dist(new UniformDistribution(-0.05, 0.05)).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder()
                                    .activation(Activation.TANH).nIn(2).nOut(2)
                                    .dist(new UniformDistribution(-0.05, 0.05)).build())
                    .layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder()
                                    .activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .nIn(2).nOut(2).build())
                    .build();


    final String json1 = conf1.toJson();

    final MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json1);

    final String json2 = conf1.toJson();


    TestCase.assertEquals(json1, json2);
}
 

/**
 * This method duplicates model for future use during inference
 */
protected void initializeReplicaModel() {
    if (protoModel instanceof ComputationGraph) {
        if (!rootDevice) {
            this.replicatedModel = new ComputationGraph(ComputationGraphConfiguration
                    .fromJson(((ComputationGraph) protoModel).getConfiguration().toJson()));
            this.replicatedModel.init();

            synchronized (locker) {
                this.replicatedModel.setParams(protoModel.params().unsafeDuplication(true));

                Nd4j.getExecutioner().commit();
            }
        } else {
            this.replicatedModel = protoModel;
        }
    } else if (protoModel instanceof MultiLayerNetwork) {
        if (!rootDevice) {
            this.replicatedModel = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(
                    ((MultiLayerNetwork) protoModel).getLayerWiseConfigurations().toJson()));
            this.replicatedModel.init();

            synchronized (locker) {
                this.replicatedModel.setParams(protoModel.params().unsafeDuplication(true));

                Nd4j.getExecutioner().commit();
            }
        } else {
            this.replicatedModel = protoModel;
        }
    }
}
 

@Test
public void testSpatialDropoutJSON(){

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new DropoutLayer.Builder(new SpatialDropout(0.5)).build())
            .build();

    String asJson = conf.toJson();
    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson);

    assertEquals(conf, fromJson);
}
 

@Test
    public void testCnn1DWithCropping1D() {
        Nd4j.getRandom().setSeed(1337);

        int[] minibatchSizes = {1, 3};
        int length = 7;
        int convNIn = 2;
        int convNOut1 = 3;
        int convNOut2 = 4;
        int finalNOut = 4;


        int[] kernels = {1, 2, 4};
        int stride = 1;

        int padding = 0;
        int cropping = 1;
        int croppedLength = length - 2 * cropping;

        Activation[] activations = {Activation.SIGMOID};
        SubsamplingLayer.PoolingType[] poolingTypes =
                new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

        for (Activation afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    for (int kernel : kernels) {
                        INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
                        INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, croppedLength);
                        for (int i = 0; i < minibatchSize; i++) {
                            for (int j = 0; j < croppedLength; j++) {
                                labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
                            }
                        }

                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .updater(new NoOp())
                                .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                                .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                        .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                        .build())
                                .layer(new Cropping1D.Builder(cropping).build())
                                .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                        .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
                                        .build())
                                .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                .setInputType(InputType.recurrent(convNIn, length)).build();

                        String json = conf.toJson();
                        MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                        assertEquals(conf, c2);

                        MultiLayerNetwork net = new MultiLayerNetwork(conf);
                        net.init();

                        String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                                + afn + ", kernel = " + kernel;

                        if (PRINT_RESULTS) {
                            System.out.println(msg);
//                            for (int j = 0; j < net.getnLayers(); j++)
//                                System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                        }

                        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                        assertTrue(msg, gradOK);

                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
 

@Test
    public void testCustomOutputLayerMLN() {
        //Second: let's create a MultiLayerCofiguration with one, and check JSON and YAML config actually works...
        MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().seed(12345).list()
                                        .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build())
                                        .layer(1, new CustomOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                        .activation(Activation.SOFTMAX)
                                                        .nIn(10).nOut(10).build())
                                        .build();

        String json = conf.toJson();
        String yaml = conf.toYaml();

//        System.out.println(json);

        MultiLayerConfiguration confFromJson = MultiLayerConfiguration.fromJson(json);
        assertEquals(conf, confFromJson);

        MultiLayerConfiguration confFromYaml = MultiLayerConfiguration.fromYaml(yaml);
        assertEquals(conf, confFromYaml);

        //Third: check initialization
        Nd4j.getRandom().setSeed(12345);
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        assertTrue(net.getLayer(1) instanceof CustomOutputLayerImpl);

        //Fourth: compare to an equivalent standard output layer (should be identical)
        MultiLayerConfiguration conf2 =
                        new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER)
                                        .list()
                                        .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1,
                                                        new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                                .activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
                                        .build();
        Nd4j.getRandom().setSeed(12345);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net2.init();

        assertEquals(net2.params(), net.params());

        INDArray testFeatures = Nd4j.rand(1, 10);
        INDArray testLabels = Nd4j.zeros(1, 10);
        testLabels.putScalar(0, 3, 1.0);
        DataSet ds = new DataSet(testFeatures, testLabels);

        assertEquals(net2.output(testFeatures), net.output(testFeatures));
        assertEquals(net2.score(ds), net.score(ds), 1e-6);
    }
 

@Override
public Iterator<Tuple2<K, Double>> call(Iterator<Tuple2<K, DataSet>> iterator) throws Exception {
    if (!iterator.hasNext()) {
        return Collections.emptyIterator();
    }

    MultiLayerNetwork network = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(jsonConfig.getValue()));
    network.init();
    INDArray val = params.value().unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException(
                        "Network did not have same number of parameters as the broadcast set parameters");
    network.setParameters(val);

    List<Tuple2<K, Double>> ret = new ArrayList<>();

    List<DataSet> collect = new ArrayList<>(batchSize);
    List<K> collectKey = new ArrayList<>(batchSize);
    int totalCount = 0;
    while (iterator.hasNext()) {
        collect.clear();
        collectKey.clear();
        int nExamples = 0;
        while (iterator.hasNext() && nExamples < batchSize) {
            Tuple2<K, DataSet> t2 = iterator.next();
            DataSet ds = t2._2();
            int n = ds.numExamples();
            if (n != 1)
                throw new IllegalStateException("Cannot score examples with one key per data set if "
                                + "data set contains more than 1 example (numExamples: " + n + ")");
            collect.add(ds);
            collectKey.add(t2._1());
            nExamples += n;
        }
        totalCount += nExamples;

        DataSet data = DataSet.merge(collect);


        INDArray scores = network.scoreExamples(data, addRegularization);
        double[] doubleScores = scores.data().asDouble();

        for (int i = 0; i < doubleScores.length; i++) {
            ret.add(new Tuple2<>(collectKey.get(i), doubleScores[i]));
        }
    }

    Nd4j.getExecutioner().commit();

    if (log.isDebugEnabled()) {
        log.debug("Scored {} examples ", totalCount);
    }

    return ret.iterator();
}
 

@Test
public void testFrozenWithBackpropLayerInstantiation() {
    //We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if
    // they were initialized via the builder
    MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).list()
            .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build())
            .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build())
            .layer(2, new OutputLayer.Builder(
                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10)
                    .nOut(10).build())
            .build();

    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0,
            new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(10).nOut(10)
                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()))
            .layer(1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(
                    new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH)
                            .weightInit(WeightInit.XAVIER).build()))
            .layer(2, new OutputLayer.Builder(
                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10)
                    .nOut(10).build())
            .build();

    MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
    net1.init();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    assertEquals(net1.params(), net2.params());


    String json = conf2.toJson();
    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json);

    assertEquals(conf2, fromJson);

    MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson);
    net3.init();

    INDArray input = Nd4j.rand(10, 10);

    INDArray out2 = net2.output(input);
    INDArray out3 = net3.output(input);

    assertEquals(out2, out3);
}
 

@Test
public void testCustomUpdater() {

    //Create a simple custom updater, equivalent to SGD updater

    double lr = 0.03;

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345)
                    .activation(Activation.TANH).updater(new CustomIUpdater(lr)) //Specify custom IUpdater
                    .list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build())
                    .layer(1, new OutputLayer.Builder().nIn(10).nOut(10)
                                    .lossFunction(LossFunctions.LossFunction.MSE).build())
                    .build();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345)
                    .activation(Activation.TANH).updater(new Sgd(lr)).list()
                    .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder()
                                    .nIn(10).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build())
                    .build();

    //First: Check updater config
    assertTrue(((BaseLayer) conf1.getConf(0).getLayer()).getIUpdater() instanceof CustomIUpdater);
    assertTrue(((BaseLayer) conf1.getConf(1).getLayer()).getIUpdater() instanceof CustomIUpdater);
    assertTrue(((BaseLayer) conf2.getConf(0).getLayer()).getIUpdater() instanceof Sgd);
    assertTrue(((BaseLayer) conf2.getConf(1).getLayer()).getIUpdater() instanceof Sgd);

    CustomIUpdater u0_0 = (CustomIUpdater) ((BaseLayer) conf1.getConf(0).getLayer()).getIUpdater();
    CustomIUpdater u0_1 = (CustomIUpdater) ((BaseLayer) conf1.getConf(1).getLayer()).getIUpdater();
    assertEquals(lr, u0_0.getLearningRate(), 1e-6);
    assertEquals(lr, u0_1.getLearningRate(), 1e-6);

    Sgd u1_0 = (Sgd) ((BaseLayer) conf2.getConf(0).getLayer()).getIUpdater();
    Sgd u1_1 = (Sgd) ((BaseLayer) conf2.getConf(1).getLayer()).getIUpdater();
    assertEquals(lr, u1_0.getLearningRate(), 1e-6);
    assertEquals(lr, u1_1.getLearningRate(), 1e-6);


    //Second: check JSON
    String asJson = conf1.toJson();
    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson);
    assertEquals(conf1, fromJson);

    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
    net1.init();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();


    //Third: check gradients are equal
    INDArray in = Nd4j.rand(5, 10);
    INDArray labels = Nd4j.rand(5, 10);

    net1.setInput(in);
    net2.setInput(in);

    net1.setLabels(labels);
    net2.setLabels(labels);

    net1.computeGradientAndScore();
    net2.computeGradientAndScore();;

    assertEquals(net1.getFlattenedGradients(), net2.getFlattenedGradients());
}
 

@Test
    public void testCnn1DWithSubsampling1D() {
        Nd4j.getRandom().setSeed(12345);

        int[] minibatchSizes = {1, 3};
        int length = 7;
        int convNIn = 2;
        int convNOut1 = 3;
        int convNOut2 = 4;
        int finalNOut = 4;

        int[] kernels = {1, 2, 4};
        int stride = 1;
        int padding = 0;
        int pnorm = 2;

        Activation[] activations = {Activation.SIGMOID, Activation.TANH};
        SubsamplingLayer.PoolingType[] poolingTypes =
                new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

        for (Activation afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    for (int kernel : kernels) {
                        INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
                        INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
                        for (int i = 0; i < minibatchSize; i++) {
                            for (int j = 0; j < length; j++) {
                                labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
                            }
                        }

                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .updater(new NoOp())
                                .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                                .layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                        .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                        .build())
                                .layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                        .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
                                        .build())
                                .layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
                                        .stride(stride).padding(padding).pnorm(pnorm).build())
                                .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                .setInputType(InputType.recurrent(convNIn, length)).build();

                        String json = conf.toJson();
                        MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                        assertEquals(conf, c2);

                        MultiLayerNetwork net = new MultiLayerNetwork(conf);
                        net.init();

                        String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                                + afn + ", kernel = " + kernel;

                        if (PRINT_RESULTS) {
                            System.out.println(msg);
//                            for (int j = 0; j < net.getnLayers(); j++)
//                                System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                        }

                        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                        assertTrue(msg, gradOK);
                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
 

@Test
    public void testCnn1DWithZeroPadding1D() {
        Nd4j.getRandom().setSeed(1337);

        int[] minibatchSizes = {1, 3};
        int length = 7;
        int convNIn = 2;
        int convNOut1 = 3;
        int convNOut2 = 4;
        int finalNOut = 4;


        int[] kernels = {1, 2, 4};
        int stride = 1;
        int pnorm = 2;

        int padding = 0;
        int zeroPadding = 2;
        int paddedLength = length + 2 * zeroPadding;

        Activation[] activations = {Activation.SIGMOID};
        SubsamplingLayer.PoolingType[] poolingTypes =
                new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

        for (Activation afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    for (int kernel : kernels) {
                        INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
                        INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, paddedLength);
                        for (int i = 0; i < minibatchSize; i++) {
                            for (int j = 0; j < paddedLength; j++) {
                                labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
                            }
                        }

                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .updater(new NoOp())
                                .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                                .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                        .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                        .build())
                                .layer(new ZeroPadding1DLayer.Builder(zeroPadding).build())
                                .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                        .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
                                        .build())
                                .layer(new ZeroPadding1DLayer.Builder(0).build())
                                .layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
                                        .stride(stride).padding(padding).pnorm(pnorm).build())
                                .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                .setInputType(InputType.recurrent(convNIn, length)).build();

                        String json = conf.toJson();
                        MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                        assertEquals(conf, c2);

                        MultiLayerNetwork net = new MultiLayerNetwork(conf);
                        net.init();

                        String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                                + afn + ", kernel = " + kernel;

                        if (PRINT_RESULTS) {
                            System.out.println(msg);
//                            for (int j = 0; j < net.getnLayers(); j++)
//                                System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                        }

                        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                        assertTrue(msg, gradOK);
                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
 

@Test
    public void testCnn1DWithLocallyConnected1D() {
        Nd4j.getRandom().setSeed(1337);

        int[] minibatchSizes = {2, 3};
        int length = 7;
        int convNIn = 2;
        int convNOut1 = 3;
        int convNOut2 = 4;
        int finalNOut = 4;

        int[] kernels = {1};
        int stride = 1;
        int padding = 0;

        Activation[] activations = {Activation.SIGMOID};

        for (Activation afn : activations) {
            for (int minibatchSize : minibatchSizes) {
                for (int kernel : kernels) {
                    INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
                    INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
                    for (int i = 0; i < minibatchSize; i++) {
                        for (int j = 0; j < length; j++) {
                            labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
                        }
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .dataType(DataType.DOUBLE)
                            .updater(new NoOp())
                            .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                            .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                    .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                    .build())
                            .layer(new LocallyConnected1D.Builder().activation(afn).kernelSize(kernel)
                                    .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2).hasBias(false)
                                    .build())
                            .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                            .setInputType(InputType.recurrent(convNIn, length)).build();

                    String json = conf.toJson();
                    MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                    assertEquals(conf, c2);

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "Minibatch=" + minibatchSize + ", activationFn="
                            + afn + ", kernel = " + kernel;

                    if (PRINT_RESULTS) {
                        System.out.println(msg);
//                        for (int j = 0; j < net.getnLayers(); j++)
//                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);

                    TestUtils.testModelSerialization(net);
                }

            }
        }
    }
 

@Test
public void testDeconv3d() {
    Nd4j.getRandom().setSeed(12345);
    // Note: we checked this with a variety of parameters, but it takes a lot of time.
    int[] depths = {8, 8, 9};
    int[] heights = {8, 9, 9};
    int[] widths = {8, 8, 9};


    int[][] kernels = {{2, 2, 2}, {3, 3, 3}, {2, 3, 2}};
    int[][] strides = {{1, 1, 1}, {1, 1, 1}, {2, 2, 2}};

    Activation[] activations = {Activation.SIGMOID, Activation.TANH, Activation.IDENTITY};

    ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same, ConvolutionMode.Same};
    int[] mbs = {1, 3, 2};
    Convolution3D.DataFormat[] dataFormats = new Convolution3D.DataFormat[]{Convolution3D.DataFormat.NCDHW, Convolution3D.DataFormat.NDHWC, Convolution3D.DataFormat.NCDHW};

    int convNIn = 2;
    int finalNOut = 2;
    int[] deconvOut = {2, 3, 4};

    for (int i = 0; i < activations.length; i++) {
        Activation afn = activations[i];
        int miniBatchSize = mbs[i];
        int depth = depths[i];
        int height = heights[i];
        int width = widths[i];
        ConvolutionMode mode = modes[i];
        int[] kernel = kernels[i];
        int[] stride = strides[i];
        Convolution3D.DataFormat df = dataFormats[i];
        int dOut = deconvOut[i];

        INDArray input;
        if (df == Convolution3D.DataFormat.NDHWC) {
            input = Nd4j.rand(new int[]{miniBatchSize, depth, height, width, convNIn});
        } else {
            input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width});
        }
        INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
        for (int j = 0; j < miniBatchSize; j++) {
            labels.putScalar(new int[]{j, j % finalNOut}, 1.0);
        }

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .dataType(DataType.DOUBLE)
                .updater(new NoOp())
                .weightInit(new NormalDistribution(0, 0.1))
                .list()
                .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel)
                        .stride(stride).nIn(convNIn).nOut(dOut).hasBias(false)
                        .convolutionMode(mode).dataFormat(df)
                        .build())
                .layer(1, new Deconvolution3D.Builder().activation(afn).kernelSize(kernel)
                        .stride(stride).nOut(dOut).hasBias(false)
                        .convolutionMode(mode).dataFormat(df)
                        .build())
                .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                        .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build();

        String json = conf.toJson();
        MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
        assertEquals(conf, c2);

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        String msg = "DataFormat = " + df + ", minibatch size = " + miniBatchSize + ", activationFn=" + afn
                + ", kernel = " + Arrays.toString(kernel) + ", stride = "
                + Arrays.toString(stride) + ", mode = " + mode.toString()
                + ", input depth " + depth + ", input height " + height
                + ", input width " + width;

        if (PRINT_RESULTS) {
            log.info(msg);
        }

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
                .labels(labels).subset(true).maxPerParam(64));

        assertTrue(msg, gradOK);

        TestUtils.testModelSerialization(net);
    }
}
 

@Test
    public void testCnn3DUpsampling() {
        Nd4j.getRandom().setSeed(42);

        int depth = 2;
        int height = 2;
        int width = 2;


        int[] minibatchSizes = {3};
        int convNIn = 2;
        int convNOut = 4;
        int denseNOut = 5;
        int finalNOut = 42;


        int[] upsamplingSize = {2, 2, 2};

        Activation[] activations = {Activation.SIGMOID};


        ConvolutionMode[] modes = {ConvolutionMode.Truncate};

        for (Activation afn : activations) {
            for (int miniBatchSize : minibatchSizes) {
                for (ConvolutionMode mode : modes) {
                    for(Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) {

                        int outDepth = depth * upsamplingSize[0];
                        int outHeight = height * upsamplingSize[1];
                        int outWidth = width * upsamplingSize[2];

                        INDArray input = df == Convolution3D.DataFormat.NCDHW ? Nd4j.rand(miniBatchSize, convNIn, depth, height, width) : Nd4j.rand(miniBatchSize, depth, height, width, convNIn);
                        INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
                        for (int i = 0; i < miniBatchSize; i++) {
                            labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
                        }

                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL)
                                .dist(new NormalDistribution(0, 1))
                                .seed(12345)
                                .list()
                                .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
                                        .nIn(convNIn).nOut(convNOut).hasBias(false)
                                        .convolutionMode(mode).dataFormat(df)
                                        .build())
                                .layer(1, new Upsampling3D.Builder(upsamplingSize[0]).dataFormat(df).build())
                                .layer(2, new DenseLayer.Builder().nOut(denseNOut).build())
                                .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                .inputPreProcessor(2,
                                        new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,
                                                convNOut, true))
                                .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build();

                        String json = conf.toJson();
                        MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                        assertEquals(conf, c2);

                        MultiLayerNetwork net = new MultiLayerNetwork(conf);
                        net.init();

                        String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
                                + ", kernel = " + Arrays.toString(upsamplingSize) + ", mode = " + mode.toString()
                                + ", input depth " + depth + ", input height " + height
                                + ", input width " + width;

                        if (PRINT_RESULTS) {
                            log.info(msg);
//                            for (int j = 0; j < net.getnLayers(); j++) {
//                                log.info("Layer " + j + " # params: " + net.getLayer(j).numParams());
//                            }
                        }

                        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS,
                                DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS,
                                RETURN_ON_FIRST_FAILURE, input, labels);

                        assertTrue(msg, gradOK);

                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
 

@Test
public void testCnn3DPooling() {
    Nd4j.getRandom().setSeed(42);

    int depth = 4;
    int height = 4;
    int width = 4;


    int[] minibatchSizes = {3};
    int convNIn = 2;
    int convNOut = 4;
    int denseNOut = 5;
    int finalNOut = 42;

    int[] kernel = {2, 2, 2};

    Activation[] activations = {Activation.SIGMOID};

    Subsampling3DLayer.PoolingType[] poolModes = {Subsampling3DLayer.PoolingType.AVG};

    ConvolutionMode[] modes = {ConvolutionMode.Truncate};

    for (Activation afn : activations) {
        for (int miniBatchSize : minibatchSizes) {
            for (Subsampling3DLayer.PoolingType pool : poolModes) {
                for (ConvolutionMode mode : modes) {
                    for (Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) {

                        int outDepth = depth / kernel[0];
                        int outHeight = height / kernel[1];
                        int outWidth = width / kernel[2];

                        INDArray input = Nd4j.rand(
                                df == Convolution3D.DataFormat.NCDHW ? new int[]{miniBatchSize, convNIn, depth, height, width}
                                        : new int[]{miniBatchSize, depth, height, width, convNIn});
                        INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
                        for (int i = 0; i < miniBatchSize; i++) {
                            labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
                        }

                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .updater(new NoOp())
                                .weightInit(WeightInit.XAVIER)
                                .dist(new NormalDistribution(0, 1))
                                .list()
                                .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
                                        .nIn(convNIn).nOut(convNOut).hasBias(false)
                                        .convolutionMode(mode).dataFormat(df)
                                        .build())
                                .layer(1, new Subsampling3DLayer.Builder(kernel)
                                        .poolingType(pool).convolutionMode(mode).dataFormat(df).build())
                                .layer(2, new DenseLayer.Builder().nOut(denseNOut).build())
                                .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                .inputPreProcessor(2,
                                        new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,convNOut, df))
                                .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build();

                        String json = conf.toJson();
                        MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                        assertEquals(conf, c2);

                        MultiLayerNetwork net = new MultiLayerNetwork(conf);
                        net.init();

                        String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
                                + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString()
                                + ", input depth " + depth + ", input height " + height
                                + ", input width " + width + ", dataFormat=" + df;

                        if (PRINT_RESULTS) {
                            log.info(msg);
                        }

                        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS,
                                DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS,
                                RETURN_ON_FIRST_FAILURE, input, labels);

                        assertTrue(msg, gradOK);

                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
}
 

@Test
    public void testCnn3DZeroPadding() {
        Nd4j.getRandom().setSeed(42);

        int depth = 4;
        int height = 4;
        int width = 4;


        int[] minibatchSizes = {3};
        int convNIn = 2;
        int convNOut1 = 3;
        int convNOut2 = 4;
        int denseNOut = 5;
        int finalNOut = 42;


        int[] kernel = {2, 2, 2};
        int[] zeroPadding = {1, 1, 2, 2, 3, 3};

        Activation[] activations = {Activation.SIGMOID};

        ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same};

        for (Activation afn : activations) {
            for (int miniBatchSize : minibatchSizes) {
                for (ConvolutionMode mode : modes) {

                    int outDepth = mode == ConvolutionMode.Same ?
                            depth : (depth - kernel[0]) + 1;
                    int outHeight = mode == ConvolutionMode.Same ?
                            height : (height - kernel[1]) + 1;
                    int outWidth = mode == ConvolutionMode.Same ?
                            width : (width - kernel[2]) + 1;

                    outDepth += zeroPadding[0] + zeroPadding[1];
                    outHeight += zeroPadding[2] + zeroPadding[3];
                    outWidth += zeroPadding[4] + zeroPadding[5];

                    INDArray input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width});
                    INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
                    for (int i = 0; i < miniBatchSize; i++) {
                        labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .dataType(DataType.DOUBLE)
                            .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL)
                            .dist(new NormalDistribution(0, 1))
                            .list()
                            .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel)
                                    .nIn(convNIn).nOut(convNOut1).hasBias(false)
                                    .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
                                    .build())
                            .layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
                                    .nIn(convNOut1).nOut(convNOut2).hasBias(false)
                                    .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
                                    .build())
                            .layer(2, new ZeroPadding3DLayer.Builder(zeroPadding).build())
                            .layer(3, new DenseLayer.Builder().nOut(denseNOut).build())
                            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                            .inputPreProcessor(3,
                                    new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,
                                            convNOut2, true))
                            .setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build();

                    String json = conf.toJson();
                    MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                    assertEquals(conf, c2);

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
                            + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString()
                            + ", input depth " + depth + ", input height " + height
                            + ", input width " + width;

                    if (PRINT_RESULTS) {
                        log.info(msg);
//                        for (int j = 0; j < net.getnLayers(); j++) {
//                            log.info("Layer " + j + " # params: " + net.getLayer(j).numParams());
//                        }
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
                            .labels(labels).subset(true).maxPerParam(512));

                    assertTrue(msg, gradOK);

                    TestUtils.testModelSerialization(net);
                }

            }
        }
    }
 

/**
 * Validate whether the file represents a valid MultiLayerNetwork saved previously with {@link MultiLayerNetwork#save(File)}
 * or {@link ModelSerializer#writeModel(Model, File, boolean)}, to be read with {@link MultiLayerNetwork#load(File, boolean)}
 *
 * @param f File that should represent an saved MultiLayerNetwork
 * @return Result of validation
 */
public static ValidationResult validateMultiLayerNetwork(@NonNull File f){

    List<String> requiredEntries = Arrays.asList(ModelSerializer.CONFIGURATION_JSON, ModelSerializer.COEFFICIENTS_BIN);     //TODO no-params models... might be OK to have no params, but basically useless in practice

    ValidationResult vr = Nd4jCommonValidator.isValidZipFile(f, false, requiredEntries);
    if(vr != null && !vr.isValid()) {
        vr.setFormatClass(MultiLayerNetwork.class);
        vr.setFormatType("MultiLayerNetwork");
        return vr;
    }

    //Check that configuration (JSON) can actually be deserialized correctly...
    String config;
    try(ZipFile zf = new ZipFile(f)){
        ZipEntry ze = zf.getEntry(ModelSerializer.CONFIGURATION_JSON);
        config = IOUtils.toString(new BufferedReader(new InputStreamReader(zf.getInputStream(ze), StandardCharsets.UTF_8)));
    } catch (IOException e){
        return ValidationResult.builder()
                .formatType("MultiLayerNetwork")
                .formatClass(MultiLayerNetwork.class)
                .valid(false)
                .path(Nd4jCommonValidator.getPath(f))
                .issues(Collections.singletonList("Unable to read configuration from model zip file"))
                .exception(e)
                .build();
    }

    try{
        MultiLayerConfiguration.fromJson(config);
    } catch (Throwable t){
        return ValidationResult.builder()
                .formatType("MultiLayerNetwork")
                .formatClass(MultiLayerNetwork.class)
                .valid(false)
                .path(Nd4jCommonValidator.getPath(f))
                .issues(Collections.singletonList("Zip file JSON model configuration does not appear to represent a valid MultiLayerConfiguration"))
                .exception(t)
                .build();
    }

    //TODO should we check params too?

    return ValidationResult.builder()
            .formatType("MultiLayerNetwork")
            .formatClass(MultiLayerNetwork.class)
            .valid(true)
            .path(Nd4jCommonValidator.getPath(f))
            .build();
}