org.deeplearning4j.nn.conf.layers.SubsamplingLayer Java Examples

@Test
public void testSubSampleMeanActivate() throws Exception {
    INDArray containedExpectedOut =
                    Nd4j.create(new double[] {2., 4., 3., 5., 3.5, 6.5, 4.5, 8.5}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());
    INDArray containedInput = getContainedData();
    INDArray input = getData();
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG);

    INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, featureMapWidth, featureMapHeight},
                    output.shape()));
    assertEquals(nChannelsIn, output.size(1), 1e-4); // channels retained
}
 

protected void setLayerOptionsBuilder(SubsamplingLayer.Builder builder, double[] values) {
    super.setLayerOptionsBuilder(builder, values);
    if (convolutionMode != null)
        builder.convolutionMode(convolutionMode.getValue(values));
    if (poolingType != null)
        builder.poolingType(poolingType.getValue(values));
    if (dilation !=null)
        builder.dilation(dilation.getValue(values));
    if (kernelSize != null)
        builder.kernelSize(kernelSize.getValue(values));
    if (stride != null)
        builder.stride(stride.getValue(values));
    if (padding != null)
        builder.padding(padding.getValue(values));
    if(pnorm != null)
        builder.pnorm(pnorm.getValue(values));
    if(eps != null)
        builder.eps(eps.getValue(values));
}
 

@Test
public void testSubSampleLayerAvgBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 2., 3., 4., 5., 6., 7., 8.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0.25, 0.25, 0.5, 0.5, 0.25, 0.25, 0.5, 0.5,
                    0.75, 0.75, 1., 1., 0.75, 0.75, 1., 1., 1.25, 1.25, 1.5, 1.5, 1.25, 1.25, 1.5, 1.5, 1.75, 1.75,
                    2., 2., 1.75, 1.75, 2., 2.}, new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType());
    INDArray input = getContainedData();

    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG);
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertArrayEquals(expectedContainedEpsilonResult.shape(), containedOutput.getSecond().shape());

}
 

@Test
public void testSubSampleMaxActivate() throws Exception {
    INDArray containedExpectedOut =
                    Nd4j.create(new double[] {5., 7., 6., 8., 4., 7., 5., 9.}, new long[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());
    INDArray containedInput = getContainedData();
    INDArray input = getData();
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX);

    INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, featureMapWidth, featureMapHeight},
                    output.shape()));
    assertEquals(nChannelsIn, output.size(1), 1e-4); // channels retained
}
 

public static ComputationGraph getOriginalGraph(int seed){
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(seed)
            .weightInit(WeightInit.XAVIER)
            .activation(Activation.TANH)
            .convolutionMode(ConvolutionMode.Same)
            .updater(new Sgd(0.3))
            .graphBuilder()
            .addInputs("in")
            .layer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2,2).stride(1,1).build(), "in")
            .layer("1", new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build(), "0")
            .layer("2", new ConvolutionLayer.Builder().nIn(3).nOut(3).kernelSize(2,2).stride(1,1).build(), "1")
            .layer("3", new DenseLayer.Builder().nOut(64).build(), "2")
            .layer("4", new DenseLayer.Builder().nIn(64).nOut(64).build(), "3")
            .layer("5", new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build(), "4")
            .setOutputs("5")
            .setInputTypes(InputType.convolutionalFlat(28,28,1))
            .build();


    ComputationGraph net = new ComputationGraph(conf);
    net.init();
    return net;
}
 

public static MultiLayerNetwork getOriginalNet(int seed){
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(seed)
            .weightInit(WeightInit.XAVIER)
            .activation(Activation.TANH)
            .convolutionMode(ConvolutionMode.Same)
            .updater(new Sgd(0.3))
            .list()
            .layer(new ConvolutionLayer.Builder().nOut(3).kernelSize(2,2).stride(1,1).build())
            .layer(new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build())
            .layer(new ConvolutionLayer.Builder().nIn(3).nOut(3).kernelSize(2,2).stride(1,1).build())
            .layer(new DenseLayer.Builder().nOut(64).build())
            .layer(new DenseLayer.Builder().nIn(64).nOut(64).build())
            .layer(new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build())
            .setInputType(InputType.convolutionalFlat(28,28,1))
            .build();


    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    return net;
}
 

private void buildPooling2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception {
    Map<String, Object> layerConfig = new HashMap<>();
    layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_MAX_POOLING_2D());
    Map<String, Object> config = new HashMap<>();
    config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME);
    List<Integer> kernelSizeList = new ArrayList<>();
    kernelSizeList.add(KERNEL_SIZE[0]);
    kernelSizeList.add(KERNEL_SIZE[1]);
    config.put(conf.getLAYER_FIELD_POOL_SIZE(), kernelSizeList);
    List<Integer> subsampleList = new ArrayList<>();
    subsampleList.add(STRIDE[0]);
    subsampleList.add(STRIDE[1]);
    config.put(conf.getLAYER_FIELD_POOL_STRIDES(), subsampleList);
    config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID);
    layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config);
    layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion);

    SubsamplingLayer layer = new KerasPooling2D(layerConfig).getSubsampling2DLayer();
    assertEquals(LAYER_NAME, layer.getLayerName());
    assertArrayEquals(KERNEL_SIZE, layer.getKernelSize());
    assertArrayEquals(STRIDE, layer.getStride());
    assertEquals(POOLING_TYPE, layer.getPoolingType());
    assertEquals(ConvolutionMode.Truncate, layer.getConvolutionMode());
    assertArrayEquals(VALID_PADDING, layer.getPadding());
}
 

/**
 * Constructor from parsed Keras layer configuration dictionary.
 *
 * @param layerConfig           dictionary containing Keras layer configuration
 * @param enforceTrainingConfig whether to enforce training-related configuration options
 * @throws InvalidKerasConfigurationException     Invalid Keras config
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
public KerasPooling2D(Map<String, Object> layerConfig, boolean enforceTrainingConfig)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    super(layerConfig, enforceTrainingConfig);
    SubsamplingLayer.Builder builder = new SubsamplingLayer.Builder(
            KerasPoolingUtils.mapPoolingType(this.className, conf)).name(this.layerName)
            .dropOut(this.dropout)
            .convolutionMode(getConvolutionModeFromConfig(layerConfig, conf))
            .kernelSize(getKernelSizeFromConfig(layerConfig, 2, conf, kerasMajorVersion))
            .stride(getStrideFromConfig(layerConfig, 2, conf));
    int[] padding = getPaddingFromBorderModeConfig(layerConfig, 2, conf, kerasMajorVersion);
    if (padding != null)
        builder.padding(padding);
    this.layer = builder.build();
    this.vertex = null;
}
 

private void buildBlock4e(ComputationGraphConfiguration.GraphBuilder graph) {
    convolution2dAndBN(graph, "inception_4e_3x3",
            160, 640, new int[]{1, 1}, new int[]{1, 1},
            256, 160, new int[]{3, 3}, new int[]{2, 2},
            new int[]{1, 1, 1, 1}, "inception_4a");
    String rel1 = lastReluId();

    convolution2dAndBN(graph, "inception_4e_5x5",
            64, 640, new int[]{1, 1}, new int[]{1, 1},
            128, 64, new int[]{5, 5}, new int[]{2, 2},
            new int[]{2, 2, 2, 2}, "inception_4a");
    String rel2 = lastReluId();

    graph.addLayer("pool8",
            new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[]{3, 3},
                    new int[]{2, 2})
                    .convolutionMode(ConvolutionMode.Truncate)
                    .build(),
            "inception_4a");
    graph.addLayer(nextPaddingId(),
            new ZeroPaddingLayer.Builder(new int[]{0, 1, 0, 1})
                    .build(), "pool8");
    String pad1 = lastPaddingId();

    graph.addVertex("inception_4e", new MergeVertex(), rel1, rel2, pad1);
}
 

private void buildBlock5b(ComputationGraphConfiguration.GraphBuilder graph) {
    convolution2dAndBN(graph, "inception_5b_3x3",
            96, 736, new int[]{1, 1}, new int[]{1, 1},
            384, 96, new int[]{3, 3}, new int[]{1, 1},
            new int[]{1, 1, 1, 1}, "inception_5a");
    String rel1 = lastReluId();

    graph.addLayer("max2",
            new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[]{3, 3},
                    new int[]{2, 2})
                    .convolutionMode(ConvolutionMode.Truncate)
                    .build(),
            "inception_5a");
    convolution2dAndBN(graph, "inception_5b_pool",
            96, 736, new int[]{1, 1}, new int[]{1, 1},
            null, null, null, null,
            null, "max2");
    graph.addLayer(nextPaddingId(),
            zeroPadding(1), lastReluId());
    String pad1 = lastPaddingId();

    convolution2dAndBN(graph, "inception_5b_1x1",
            256, 736, new int[]{1, 1}, new int[]{1, 1},
            null, null, null, null,
            null, "inception_5a");
    String rel2 = lastReluId();

    graph.addVertex("inception_5b", new MergeVertex(), rel1, pad1, rel2);
}
 

@Test
public void testCudnnDilation(){
    //Sanity check on dilated conv execution
    int[] k = new int[]{2,3,4,5};
    int[] d = new int[]{1,2,3,4};

    for( int[] inputSize : new int[][]{{10,1,28,28}, {3,3,224,224}}) {
        for (int i = 0; i < k.length; i++) {
            for(ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Same, ConvolutionMode.Truncate}) {

                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .convolutionMode(ConvolutionMode.Same)
                        .list()
                        .layer(new ConvolutionLayer.Builder().kernelSize(k[i], k[i]).dilation(d[i], d[i]).nOut(3).build())
                        .layer(new SubsamplingLayer.Builder().kernelSize(k[i], k[i]).dilation(d[i], d[i]).build())
                        .layer(new OutputLayer.Builder().nOut(10).build())
                        .setInputType(InputType.convolutional(inputSize[3], inputSize[2], inputSize[1]))
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                INDArray in = Nd4j.create(inputSize);
                net.output(in);
            }
        }
    }
}
 

@Test
public void testCNNActivationsFrozen() throws Exception {

    int nChannels = 1;
    int outputNum = 10;
    int batchSize = 64;
    int nEpochs = 10;
    int seed = 123;

    log.info("Load data....");
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345);

    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new FrozenLayer(new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()))
            .layer(1, new FrozenLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build()))
            .layer(2, new FrozenLayer(new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()))
            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    log.info("Train model....");
    model.setListeners(new ConvolutionalIterationListener(1));

    for (int i = 0; i < nEpochs; i++) {
        model.fit(mnistTrain);
    }
}
 

@Test
    public void testLfwModel() throws Exception {
        final int numRows = 28;
        final int numColumns = 28;
        int numChannels = 3;
        int outputNum = LFWLoader.NUM_LABELS;
        int numSamples = LFWLoader.NUM_IMAGES;
        int batchSize = 2;
        int seed = 123;
        int listenerFreq = 1;

        LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples,
                        new int[] {numRows, numColumns, numChannels}, outputNum, false, true, 1.0, new Random(seed));

        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                        .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                        .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(numChannels).nOut(6)
                                        .weightInit(WeightInit.XAVIER).activation(Activation.RELU).build())
                        .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                        .stride(1, 1).build())
                        .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                        .build())
                        .setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels))
                        ;

        MultiLayerNetwork model = new MultiLayerNetwork(builder.build());
        model.init();

        model.setListeners(new ScoreIterationListener(listenerFreq));

        model.fit(lfw.next());

        DataSet dataTest = lfw.next();
        INDArray output = model.output(dataTest.getFeatures());
        Evaluation eval = new Evaluation(outputNum);
        eval.eval(dataTest.getLabels(), output);
//        System.out.println(eval.stats());
    }
 

public ComputationGraph getCNNMnistConfig() {

        int nChannels = 1; // Number of input channels
        int outputNum = 10; // The number of possible outcomes

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) // Training iterations as above
                        .l2(0.0005).weightInit(WeightInit.XAVIER)
                        .updater(new Nesterovs(0.01, 0.9))
                        .graphBuilder().addInputs("input")
                        .setInputTypes(InputType.convolutionalFlat(28, 28, 1))
                        .addLayer("0", new ConvolutionLayer.Builder(5, 5)
                                        //nIn and nOut specify channels. nIn here is the nChannels and nOut is the number of filters to be applied
                                        .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(),
                                        "input")
                        .addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                        .stride(2, 2).build(), "0")
                        .addLayer("2", new ConvolutionLayer.Builder(5, 5)
                                        //Note that nIn need not be specified in later layers
                                        .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1")
                        .addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                        .stride(2, 2).build(), "2")
                        .addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3")
                        .addLayer("output",
                                        new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder(
                                                        LossFunction.MCXENT).nOut(outputNum)
                                                                        .activation(Activation.SOFTMAX).build(),
                                        "4")
                        .setOutputs("output").build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        return graph;
    }
 

private Layer getSubsamplingLayer(SubsamplingLayer.PoolingType pooling) {
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder()
                    .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123)
                    .layer(new SubsamplingLayer.Builder(pooling, new int[] {2, 2}).build()).build();

    return conf.getLayer().instantiate(conf, null, 0, null, true, Nd4j.defaultFloatingPointType());
}
 

@Test(expected = UnsupportedOperationException.class)
public void testSubSampleLayerSumBackprop() throws Exception {
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.SUM);
    INDArray input = getData();
    layer.setInput(input, LayerWorkspaceMgr.noWorkspaces());
    layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
}
 

@Test
public void testSubSampleLayerMaxBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1.,
                    0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0.},
                    new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType());

    INDArray input = getContainedData();

    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX);
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    long depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1)); // channels retained
}
 

private static SubsamplingLayer avgPoolLayer() {
    return new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG)
            .kernelSize(2, 2)
            .stride(2, 2)
            .convolutionMode(ConvolutionMode.Same)
            .build();
}
 

private void buildBlock3c(ComputationGraphConfiguration.GraphBuilder graph) {
    convolution2dAndBN(graph, "inception_3c_3x3",
            128, 320, new int[]{1, 1}, new int[]{1, 1},
            256, 128, new int[]{3, 3}, new int[]{2, 2},
            new int[]{1, 1, 1, 1}, "inception_3b");
    String rel1 = lastReluId();

    convolution2dAndBN(graph, "inception_3c_5x5",
            32, 320, new int[]{1, 1}, new int[]{1, 1},
            64, 32, new int[]{5, 5}, new int[]{2, 2},
            new int[]{2, 2, 2, 2}, "inception_3b");
    String rel2 = lastReluId();

    graph.addLayer("pool7",
            new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[]{3, 3},
                    new int[]{2, 2})
                    .convolutionMode(ConvolutionMode.Truncate)
                    .build(),
            "inception_3b");

    graph.addLayer(nextPaddingId(),
            new ZeroPaddingLayer.Builder(new int[]{0, 1, 0, 1})
                    .build(), "pool7");
    String pad1 = lastPaddingId();

    graph.addVertex("inception_3c", new MergeVertex(), rel1, rel2, pad1);
}
 

private void buildBlock4a(ComputationGraphConfiguration.GraphBuilder graph) {
    convolution2dAndBN(graph, "inception_4a_3x3",
            96, 640, new int[]{1, 1}, new int[]{1, 1},
            192, 96, new int[]{3, 3}, new int[]{1, 1}
            , new int[]{1, 1, 1, 1}, "inception_3c");
    String rel1 = lastReluId();

    convolution2dAndBN(graph, "inception_4a_5x5",
            32, 640, new int[]{1, 1}, new int[]{1, 1},
            64, 32, new int[]{5, 5}, new int[]{1, 1}
            , new int[]{2, 2, 2, 2}, "inception_3c");
    String rel2 = lastReluId();

    graph.addLayer("avg7",
            new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG, new int[]{3, 3},
                    new int[]{3, 3})
                    .convolutionMode(ConvolutionMode.Truncate)
                    .build(),
            "inception_3c");
    convolution2dAndBN(graph, "inception_4a_pool",
            128, 640, new int[]{1, 1}, new int[]{1, 1},
            null, null, null, null
            , new int[]{2, 2, 2, 2}, "avg7");
    String pad1 = lastPaddingId();

    convolution2dAndBN(graph, "inception_4a_1x1",
            256, 640, new int[]{1, 1}, new int[]{1, 1},
            null, null, null, null
            , null, "inception_3c");
    String rel4 = lastReluId();
    graph.addVertex("inception_4a", new MergeVertex(), rel1, rel2, rel4, pad1);

}
 

public static ComputationGraphConfiguration.GraphBuilder addLayers(ComputationGraphConfiguration.GraphBuilder graphBuilder, int layerNumber, String input, int filterSize, int nIn, int nOut, int poolSize, int poolStride) {
    graphBuilder
            .addLayer("convolution2d_" + layerNumber,
                    new ConvolutionLayer.Builder(filterSize,filterSize)
                            .nIn(nIn)
                            .nOut(nOut)
                            .weightInit(WeightInit.XAVIER)
                            .convolutionMode(ConvolutionMode.Same)
                            .hasBias(false)
                            .stride(1,1)
                            .activation(Activation.IDENTITY)
                            .build(),
                    input)
            .addLayer("batchnormalization_" + layerNumber,
                    new BatchNormalization.Builder()
                            .nIn(nOut).nOut(nOut)
                            .weightInit(WeightInit.XAVIER)
                            .activation(Activation.IDENTITY)
                            .build(),
                    "convolution2d_" + layerNumber)
            .addLayer("activation_" + layerNumber,
                    new ActivationLayer.Builder()
                            .activation(new ActivationLReLU(0.1))
                            .build(),
                    "batchnormalization_" + layerNumber);
    if (poolSize > 0) {
        graphBuilder
                .addLayer("maxpooling2d_" + layerNumber,
                        new SubsamplingLayer.Builder()
                                .kernelSize(poolSize, poolSize)
                                .stride(poolStride, poolStride)
                                .convolutionMode(ConvolutionMode.Same)
                                .build(),
                        "activation_" + layerNumber);
    }

    return graphBuilder;
}
 

private void buildBlock5a(ComputationGraphConfiguration.GraphBuilder graph) {
    convolution2dAndBN(graph, "inception_5a_3x3",
            96, 1024, new int[]{1, 1}, new int[]{1, 1},
            384, 96, new int[]{3, 3}, new int[]{1, 1},
            new int[]{1, 1, 1, 1}, "inception_4e");
    String relu1 = lastReluId();

    graph.addLayer("avg9",
            new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG, new int[]{3, 3},
                    new int[]{3, 3})
                    .convolutionMode(ConvolutionMode.Truncate)
                    .build(),
            "inception_4e");
    convolution2dAndBN(graph, "inception_5a_pool",
            96, 1024, new int[]{1, 1}, new int[]{1, 1},
            null, null, null, null,
            new int[]{1, 1, 1, 1}, "avg9");
    String pad1 = lastPaddingId();

    convolution2dAndBN(graph, "inception_5a_1x1",
            256, 1024, new int[]{1, 1}, new int[]{1, 1},
            null, null, null, null,
            null, "inception_4e");
    String rel3 = lastReluId();

    graph.addVertex("inception_5a", new MergeVertex(), relu1, pad1, rel3);
}
 

@Test
public void cifar10Test() throws Exception {

	/* load dataset and create a train-test-split */
	OpenmlConnector connector = new OpenmlConnector();
	DataSetDescription ds = connector.dataGet(DataSetUtils.MNIST_ID);
	File file = connector.datasetGet(ds);
	Instances data = new Instances(new BufferedReader(new FileReader(file)));
	data.setClassIndex(data.numAttributes() - 1);
	List<Instances> split = WekaUtil.getStratifiedSplit(data, 42, .02f);

	logger.info("Calculating intermediates...");
	List<INDArray> intermediate = new ArrayList<>();
	intermediate.add(DataSetUtils.mnistInstanceToMatrix(split.get(0).get(0)));
	logger.info("Finished intermediate calculations.");

	PretrainedNNFilter filter = ImageUtils.getPretrainedNNFilterByName("VGG16", 12, intermediate.get(0).shape());

	logger.debug("Selected layer: " + filter.getCompGraph().getLayer(5).toString());

	DataSet result = filter.applyFilter(new DataSet(split.get(0), intermediate), false);
	logger.info(Arrays.toString(result.getIntermediateInstances().get(0).shape()));

	MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(0).list().layer(0, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).build()).build();

	MultiLayerNetwork mln = new MultiLayerNetwork(conf);
	mln.init();
	System.out.println(result.getIntermediateInstances().get(0).mean(3));
	INDArray prediction = mln.output(result.getIntermediateInstances().get(0));
	logger.debug("Prediction shape: " + Arrays.toString(prediction.shape()));
	logger.debug("Prediction mean of element 3: {}", prediction.mean(3));
	Assert.assertFalse(prediction.isEmpty());

}
 

/**
 * Returns a symmetric max pooling networking consisting only of a max pooling
 * layer.
 *
 * @param kernelSize Kernel size of the max pooling layer
 * @param stride     Stride of the max pooling layer
 * @return Returns the constructed and initialized network
 */
static MultiLayerNetwork getMaxPoolNetworkSymmetricWithCustomKernelStride(final int kernelSize,
                                                                                 final int stride) {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(0).list()
            .layer(0,
                    new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                            .kernelSize(kernelSize, kernelSize).stride(stride, stride).build())
            .build();
    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();
    return mln;
}
 

/**
 * Constructs a simple max pooling network consisting only of a max pooling
 * layer.
 *
 * @return Returns the constructed and initialized network
 */
public static MultiLayerNetwork getMaxPoolNetwork() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(0).list()
            .layer(0, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).build()).build();
    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();
    return mln;
}
 

@Test
public void testParallelWrapperRun() throws Exception {

    int nChannels = 1;
    int outputNum = 10;

    // for GPU you usually want to have higher batchSize
    int batchSize = 128;
    int nEpochs = 5;
    int seed = 123;

    log.info("Load data....");
    DataSetIterator mnistTrain = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(batchSize, true, 12345), 15);
    DataSetIterator mnistTest = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(batchSize, false, 12345), 4);

    assertTrue(mnistTrain.hasNext());
    val t0 = mnistTrain.next();

    log.info("F: {}; L: {};", t0.getFeatures().shape(), t0.getLabels().shape());

    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                    .l2(0.0005)
                    //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
                    .weightInit(WeightInit.XAVIER)
                    .updater(new Nesterovs(0.01, 0.9)).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5)
                                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
                    .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                    .stride(2, 2).build())
                    .layer(2, new ConvolutionLayer.Builder(5, 5)
                                    //Note that nIn needed be specified in later layers
                                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
                    .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                    .stride(2, 2).build())
                    .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
                    .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
                    // DataSets prefetching options. Set this value with respect to number of actual devices
                    .prefetchBuffer(24)

                    // set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
                    .workers(2)

                    // rare averaging improves performance, but might reduce model accuracy
                    .averagingFrequency(3)

                    // if set to TRUE, on every averaging model score will be reported
                    .reportScoreAfterAveraging(true)

                    // optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
                    .build();

    log.info("Train model....");
    model.setListeners(new ScoreIterationListener(100));
    long timeX = System.currentTimeMillis();

    // optionally you might want to use MultipleEpochsIterator instead of manually iterating/resetting over your iterator
    //MultipleEpochsIterator mnistMultiEpochIterator = new MultipleEpochsIterator(nEpochs, mnistTrain);

    for (int i = 0; i < nEpochs; i++) {
        long time1 = System.currentTimeMillis();

        // Please note: we're feeding ParallelWrapper with iterator, not model directly
        //            wrapper.fit(mnistMultiEpochIterator);
        wrapper.fit(mnistTrain);
        long time2 = System.currentTimeMillis();
        log.info("*** Completed epoch {}, time: {} ***", i, (time2 - time1));
    }
    long timeY = System.currentTimeMillis();
    log.info("*** Training complete, time: {} ***", (timeY - timeX));

    Evaluation eval = model.evaluate(mnistTest);
    log.info(eval.stats());
    mnistTest.reset();

    double acc = eval.accuracy();
    assertTrue(String.valueOf(acc), acc > 0.5);

    wrapper.shutdown();
}
 

@Test
public void testWrapper() throws Exception {
    int nChannels = 1;
    int outputNum = 10;

    // for GPU you usually want to have higher batchSize
    int batchSize = 128;
    int seed = 123;

    log.info("Load data....");
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, 1000);
    DataSetIterator mnistTest = new MnistDataSetIterator(batchSize, false, 12345);

    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                    .l2(0.0005)
                    .weightInit(WeightInit.XAVIER)
                    .updater(new Nesterovs(0.01, 0.9)).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5)
                                    //nIn and nOut specify channels. nIn here is the nChannels and nOut is the number of filters to be applied
                                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
                    .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                    .stride(2, 2).build())
                    .layer(2, new ConvolutionLayer.Builder(5, 5)
                                    //Note that nIn needed be specified in later layers
                                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
                    .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                    .stride(2, 2).build())
                    .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
                    .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    ParallelWrapper parameterServerParallelWrapper =
                    new ParallelWrapper.Builder(model)
                            .workers(Math.min(4, Runtime.getRuntime().availableProcessors()))
                            .trainerFactory(new ParameterServerTrainerContext())
                                    .reportScoreAfterAveraging(true).prefetchBuffer(3).build();
    parameterServerParallelWrapper.fit(mnistTrain);

    Thread.sleep(2000);
    parameterServerParallelWrapper.close();



}
 

@Test
public void testEqualUuid1() {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
            .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(2, new ConvolutionLayer.Builder(5, 5)
                    //Note that nIn needed be specified in later layers
                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
            .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
            // DataSets prefetching options. Set this value with respect to number of actual devices
            .prefetchBuffer(24)

            // set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
            .workers(2)

            // rare averaging improves performance, but might reduce model accuracy
            .averagingFrequency(3)

            // if set to TRUE, on every averaging model score will be reported
            .reportScoreAfterAveraging(true)

            // optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
            .build();

    val context = new DefaultTrainerContext();
    val trainer = context.create("alpha", 3, model, 0, true, wrapper, WorkspaceMode.NONE, 3);

    assertEquals("alpha_thread_3", trainer.getUuid());
}
 

@Override
public SubsamplingLayer getValue(double[] values) {
    SubsamplingLayer.Builder b = new SubsamplingLayer.Builder();
    setLayerOptionsBuilder(b, values);
    return b.build();
}
 

@Test
public void testEqualUuid1() {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
            .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(2, new ConvolutionLayer.Builder(5, 5)
                    //Note that nIn needed be specified in later layers
                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
            .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
            // DataSets prefetching options. Set this value with respect to number of actual devices
            .prefetchBuffer(24)

            // set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
            .workers(2)

            // rare averaging improves performance, but might reduce model accuracy
            .averagingFrequency(3)

            // if set to TRUE, on every averaging model score will be reported
            .reportScoreAfterAveraging(true)

            // optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
            .build();

    val trainer = new SymmetricTrainer(model, "alpha", 3, WorkspaceMode.NONE, wrapper, true);

    assertEquals("alpha_thread_3", trainer.getUuid());
}