Java Code Examples for org.deeplearning4j.nn.conf.ConvolutionMode#Same

public void computeOutputSize() {
    int nIn = (int) getNIn();

    if (inputSize == null) {
        throw new IllegalArgumentException("Input size has to be specified for locally connected layers.");
    }

    boolean nchw = format == CNN2DFormat.NCHW;

    int[] inputShape = nchw ? new int[] {1, nIn, inputSize[0], inputSize[1]} : new int[] {1, inputSize[0], inputSize[1], nIn};
    INDArray dummyInputForShapeInference = Nd4j.ones(inputShape);

    if (cm == ConvolutionMode.Same) {
        this.outputSize = ConvolutionUtils.getOutputSize(dummyInputForShapeInference, kernel, stride, null, cm,
                        dilation, format);
        this.padding = ConvolutionUtils.getSameModeTopLeftPadding(outputSize, inputSize, kernel, stride, dilation);
        this.paddingBr = ConvolutionUtils.getSameModeBottomRightPadding(outputSize, inputSize, kernel, stride, dilation);
    } else {
        this.outputSize = ConvolutionUtils.getOutputSize(dummyInputForShapeInference, kernel, stride, padding, cm,
                        dilation, format);
    }
}
 

@Test
public void testLocallyConnected() {
    try {
        for (boolean helpers : new boolean[]{false, true}) {
            for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) {
                Nd4j.getRandom().setSeed(12345);
                Nd4j.getEnvironment().allowHelpers(helpers);
                String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")";
                System.out.println(" --- " + msg + " ---");

                INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12);
                INDArray labels = TestUtils.randomOneHot(2, 10);

                TestCase tc = TestCase.builder()
                        .msg(msg)
                        .net1(getLocallyConnectedNet(CNN2DFormat.NCHW, true, cm))
                        .net2(getLocallyConnectedNet(CNN2DFormat.NCHW, false, cm))
                        .net3(getLocallyConnectedNet(CNN2DFormat.NHWC, true, cm))
                        .net4(getLocallyConnectedNet(CNN2DFormat.NHWC, false, cm))
                        .inNCHW(inNCHW)
                        .labelsNCHW(labels)
                        .labelsNHWC(labels)
                        .testLayerIdx(1)
                        .helpers(helpers)
                        .build();

                testHelper(tc);
            }
        }
    } finally {
        Nd4j.getEnvironment().allowHelpers(true);
    }
}
 

/**
 * Given a mask array for a 1D CNN layer of shape [minibatch, sequenceLength], reduce the mask according to the 1D CNN layer configuration.
 * Unlike RNN layers, 1D CNN layers may down-sample the data; consequently, we need to down-sample the mask array
 * in the same way, to maintain the correspondence between the masks and the output activations
 *
 * @param in       Input size
 * @param kernel   Kernel size
 * @param stride   Stride
 * @param padding  Padding
 * @param dilation Dilation
 * @param cm       Convolution mode
 * @return Reduced mask
 */
public static INDArray cnn1dMaskReduction(INDArray in, int kernel, int stride, int padding, int dilation, ConvolutionMode cm){
    Preconditions.checkState(in.rank()==2, "Rank must be 2 for cnn1d mask array - shape ", in.shape());
    if((cm == ConvolutionMode.Same || cm == ConvolutionMode.Causal) && stride == 1 ){
        return in;
    }

    if(!Shape.hasDefaultStridesForShape(in)){
        in = in.dup();
    }

    INDArray reshaped4d = in.reshape(in.size(0), 1, in.size(1), 1);

    int[] outSize;
    int[] pad = null;
    int[] k = new int[]{kernel,1};
    int[] s = new int[]{stride, 1};
    int[] d = new int[]{dilation, 1};
    if (cm == ConvolutionMode.Same || cm == ConvolutionMode.Causal) {
        outSize = ConvolutionUtils.getOutputSize(reshaped4d, k, s, null, cm, d, CNN2DFormat.NCHW); //Also performs validation
    } else {
        pad = new int[]{padding, 0};
        outSize = ConvolutionUtils.getOutputSize(reshaped4d, k, s, pad, cm, d, CNN2DFormat.NCHW); //Also performs validation
    }
    int outH = outSize[0];

    INDArray output = Nd4j.createUninitialized(new int[]{(int)in.size(0), 1, outH, 1}, 'c');

    DynamicCustomOp op = new MaxPooling2D(reshaped4d, output, Pooling2DConfig.builder()
            .kH(k[0]).kW(k[1])
            .sH(s[0]).sW(s[1])
            .pH(pad == null ? 0 : pad[0]).pW(pad == null ? 0 : pad[1])
            .dH(d[0]).dW(d[1])
            .isSameMode(cm == ConvolutionMode.Same || cm == ConvolutionMode.Causal)
            .isNHWC(false)
            .build());

    Nd4j.getExecutioner().exec(op);
    return output.reshape('c', in.size(0), outH);
}
 

/**
 * Check that the convolution mode is consistent with the padding specification
 */
public static void validateConvolutionModePadding(ConvolutionMode mode, int[] padding) {
    if (mode == ConvolutionMode.Same) {
        boolean nullPadding = true;
        for (int i : padding) {
            if (i != 0) nullPadding = false;
        }
        if (!nullPadding)
            throw new IllegalArgumentException("Padding cannot be used when using the `same' convolution mode");
    }
}
 

/**
 * Get the output size (height/width) for the given input data and CNN configuration
 *
 * @param inputData       Input data
 * @param kernel          Kernel size (height/width)
 * @param strides         Strides (height/width)
 * @param padding         Padding (height/width)
 * @param convolutionMode Convolution mode (Same, Strict, Truncate)
 * @param dilation        Kernel dilation (height/width)
 * @param format          Format for input activations
 * @return Output size: int[2] with output height/width
 */
public static int[] getOutputSize(INDArray inputData, int[] kernel, int[] strides, int[] padding,
                                  ConvolutionMode convolutionMode, int[] dilation, CNN2DFormat format) {
    int hDim = 2;
    int wDim = 3;
    if(format == CNN2DFormat.NHWC){
        hDim = 1;
        wDim = 2;
    }

    if (inputData.size(hDim) > Integer.MAX_VALUE || inputData.size(wDim) > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();
    int inH = (int) inputData.size(hDim);
    int inW = (int) inputData.size(wDim);

    //Determine the effective kernel size, accounting for dilation
    //http://deeplearning.net/software/theano/tutorial/conv_arithmetic.html#dilated-convolutions
    int[] eKernel = effectiveKernelSize(kernel, dilation);
    boolean atrous = (eKernel == kernel);

    int[] inShape = new int[]{inH, inW};
    validateShapes(inputData, eKernel, strides, padding, convolutionMode, dilation, inShape, atrous);

    if (convolutionMode == ConvolutionMode.Same || convolutionMode == ConvolutionMode.Causal) {

        int outH = (int) Math.ceil(inH / ((double) strides[0]));
        int outW = (int) Math.ceil(inW / ((double) strides[1]));

        return new int[]{outH, outW};
    }

    int hOut = (inH - eKernel[0] + 2 * padding[0]) / strides[0] + 1;
    int wOut = (inW - eKernel[1] + 2 * padding[1]) / strides[1] + 1;

    return new int[]{hOut, wOut};
}
 

/**
 * Check that the convolution mode is consistent with the padding specification
 */
public static void validateConvolutionModePadding(ConvolutionMode mode, int padding) {
    if (mode == ConvolutionMode.Same) {
        boolean nullPadding = true;
        if (padding != 0) nullPadding = false;
        if (!nullPadding)
            throw new IllegalArgumentException("Padding cannot be used when using the `same' convolution mode");

    }
}
 

@Test
public void testLRN() {
    try {
        for (boolean helpers : new boolean[]{false, true}) {
            for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) {
                Nd4j.getRandom().setSeed(12345);
                Nd4j.getEnvironment().allowHelpers(helpers);
                String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")";
                System.out.println(" --- " + msg + " ---");

                INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12);
                INDArray labels = TestUtils.randomOneHot(2, 10);

                TestCase tc = TestCase.builder()
                        .msg(msg)
                        .net1(getLrnLayer(CNN2DFormat.NCHW, true, cm))
                        .net2(getLrnLayer(CNN2DFormat.NCHW, false, cm))
                        .net3(getLrnLayer(CNN2DFormat.NHWC, true, cm))
                        .net4(getLrnLayer(CNN2DFormat.NHWC, false, cm))
                        .inNCHW(inNCHW)
                        .labelsNCHW(labels)
                        .labelsNHWC(labels)
                        .testLayerIdx(1)
                        .helpers(helpers)
                        .build();

                testHelper(tc);
            }
        }
    } finally {
        Nd4j.getEnvironment().allowHelpers(true);
    }
}
 

@Test
public void testConv2d() {
    try {
        for (boolean helpers : new boolean[]{false, true}) {
            for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) {
                Nd4j.getRandom().setSeed(12345);
                Nd4j.getEnvironment().allowHelpers(helpers);
                String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")";
                System.out.println(" --- " + msg + " ---");

                INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12);
                INDArray labels = TestUtils.randomOneHot(2, 10);

                TestCase tc = TestCase.builder()
                        .msg(msg)
                        .net1(getConv2dNet(CNN2DFormat.NCHW, true, cm))
                        .net2(getConv2dNet(CNN2DFormat.NCHW, false, cm))
                        .net3(getConv2dNet(CNN2DFormat.NHWC, true, cm))
                        .net4(getConv2dNet(CNN2DFormat.NHWC, false, cm))
                        .inNCHW(inNCHW)
                        .labelsNCHW(labels)
                        .labelsNHWC(labels)
                        .testLayerIdx(1)
                        .helpers(helpers)
                        .build();

                testHelper(tc);
            }
        }
    } finally {
        Nd4j.getEnvironment().allowHelpers(true);
    }
}
 

/**
 * Get convolution border mode from Keras layer configuration.
 *
 * @param layerConfig dictionary containing Keras layer configuration
 * @return Border mode of convolutional layers
 * @throws InvalidKerasConfigurationException Invalid Keras configuration
 */
public static ConvolutionMode getConvolutionModeFromConfig(Map<String, Object> layerConfig,
                                                           KerasLayerConfiguration conf)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
    if (!innerConfig.containsKey(conf.getLAYER_FIELD_BORDER_MODE()))
        throw new InvalidKerasConfigurationException("Could not determine convolution border mode: no "
                + conf.getLAYER_FIELD_BORDER_MODE() + " field found");
    String borderMode = (String) innerConfig.get(conf.getLAYER_FIELD_BORDER_MODE());
    ConvolutionMode convolutionMode;
    if (borderMode.equals(conf.getLAYER_BORDER_MODE_SAME())) {
        /* Keras relies upon the Theano and TensorFlow border mode definitions and operations:
         * TH: http://deeplearning.net/software/theano/library/tensor/nnet/conv.html#theano.tensor.nnet.conv.conv2d
         * TF: https://www.tensorflow.org/api_docs/python/nn/convolution#conv2d
         */
        convolutionMode = ConvolutionMode.Same;

    } else if (borderMode.equals(conf.getLAYER_BORDER_MODE_VALID()) ||
            borderMode.equals(conf.getLAYER_BORDER_MODE_FULL())) {
        convolutionMode = ConvolutionMode.Truncate;
    } else if(borderMode.equals(conf.getLAYER_BORDER_MODE_CAUSAL())) {
        convolutionMode = ConvolutionMode.Causal;
    } else {
        throw new UnsupportedKerasConfigurationException("Unsupported convolution border mode: " + borderMode);
    }
    return convolutionMode;
}
 

@Test
    public void testCnn3DCropping() {
        Nd4j.getRandom().setSeed(42);

        int depth = 6;
        int height = 6;
        int width = 6;


        int[] minibatchSizes = {3};
        int convNIn = 2;
        int convNOut1 = 3;
        int convNOut2 = 4;
        int denseNOut = 5;
        int finalNOut = 8;


        int[] kernel = {1, 1, 1};
        int[] cropping = {0, 0, 1, 1, 2, 2};

        Activation[] activations = {Activation.SIGMOID};

        ConvolutionMode[] modes = {ConvolutionMode.Same};

        for (Activation afn : activations) {
            for (int miniBatchSize : minibatchSizes) {
                for (ConvolutionMode mode : modes) {

                    int outDepth = mode == ConvolutionMode.Same ?
                            depth : (depth - kernel[0]) + 1;
                    int outHeight = mode == ConvolutionMode.Same ?
                            height : (height - kernel[1]) + 1;
                    int outWidth = mode == ConvolutionMode.Same ?
                            width : (width - kernel[2]) + 1;

                    outDepth -= cropping[0] + cropping[1];
                    outHeight -= cropping[2] + cropping[3];
                    outWidth -= cropping[4] + cropping[5];

                    INDArray input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width});
                    INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
                    for (int i = 0; i < miniBatchSize; i++) {
                        labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .dataType(DataType.DOUBLE)
                            .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL)
                            .dist(new NormalDistribution(0, 1))
                            .list()
                            .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel)
                                    .nIn(convNIn).nOut(convNOut1).hasBias(false)
                                    .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
                                    .build())
                            .layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
                                    .nIn(convNOut1).nOut(convNOut2).hasBias(false)
                                    .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
                                    .build())
                            .layer(2, new Cropping3D.Builder(cropping).build())
                            .layer(3, new DenseLayer.Builder().nOut(denseNOut).build())
                            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                            .inputPreProcessor(3,
                                    new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,
                                            convNOut2, true))
                            .setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build();

                    String json = conf.toJson();
                    MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                    assertEquals(conf, c2);

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
                            + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString()
                            + ", input depth " + depth + ", input height " + height
                            + ", input width " + width;

                    if (PRINT_RESULTS) {
                        log.info(msg);
//                        for (int j = 0; j < net.getnLayers(); j++) {
//                            log.info("Layer " + j + " # params: " + net.getLayer(j).numParams());
//                        }
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS,
                            DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS,
                            RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);

                    TestUtils.testModelSerialization(net);
                }

            }
        }
    }
 

@Test
public void validateConvSubsampling() throws Exception {
    //Only run test if using nd4j-native backend
    assumeTrue(Nd4j.getBackend().getClass().getName().toLowerCase().contains("native"));
    Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
    Nd4j.getRandom().setSeed(12345);

    int[] inputSize = {-1, 3, 16, 16};

    for(int minibatch : new int[]{1,3}) {
        for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Same, ConvolutionMode.Truncate}) {
            for (int[] kernel : new int[][]{{2, 2}, {2, 3}}) {
                for (int[] stride : new int[][]{{1, 1}, {2, 2}}) {
                    for (PoolingType pt : new PoolingType[]{PoolingType.MAX, PoolingType.AVG}) {

                        inputSize[0] = minibatch;
                        INDArray f = Nd4j.rand(DataType.FLOAT, inputSize);
                        INDArray l = TestUtils.randomOneHot(minibatch, 10).castTo(DataType.FLOAT);

                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                .updater(new Adam(0.01))
                                .convolutionMode(cm)
                                .seed(12345)
                                .list()
                                .layer(new ConvolutionLayer.Builder().activation(Activation.TANH)
                                        .kernelSize(kernel)
                                        .stride(stride)
                                        .padding(0, 0)
                                        .nOut(3)
                                        .build())
                                .layer(new SubsamplingLayer.Builder()
                                        .poolingType(pt)
                                        .kernelSize(kernel)
                                        .stride(stride)
                                        .padding(0, 0)
                                        .build())
                                .layer(new ConvolutionLayer.Builder().activation(Activation.TANH)
                                        .kernelSize(kernel)
                                        .stride(stride)
                                        .padding(0, 0)
                                        .nOut(3)
                                        .build())
                                .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
                                .setInputType(InputType.convolutional(inputSize[2], inputSize[3], inputSize[1]))
                                .build();

                        MultiLayerNetwork netWith = new MultiLayerNetwork(conf.clone());
                        netWith.init();

                        MultiLayerNetwork netWithout = new MultiLayerNetwork(conf.clone());
                        netWithout.init();

                        String name = pt + ", mb=" + minibatch + ", cm=" + cm + ", kernel=" + Arrays.toString(kernel) + ", stride=" + Arrays.toString(stride);
                        LayerHelperValidationUtil.TestCase tc = LayerHelperValidationUtil.TestCase.builder()
                                .testName(name)
                                .allowHelpersForClasses(Arrays.<Class<?>>asList(org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer.class,
                                        org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class))
                                .testForward(true)
                                .testScore(true)
                                .testBackward(true)
                                .testTraining(true)
                                .features(f)
                                .labels(l)
                                .data(new SingletonDataSetIterator(new DataSet(f, l)))
                                .build();

                        System.out.println("Starting test: " + name);
                        LayerHelperValidationUtil.validateMLN(netWith, tc);
                    }
                }
            }
        }
    }
}
 

@Override
public SDVariable defineLayer(SameDiff sameDiff, SDVariable layerInput, Map<String, SDVariable> paramTable, SDVariable mask) {
    SDVariable w = paramTable.get(ConvolutionParamInitializer.WEIGHT_KEY); // (outH, featureDim, nOut)

    int outH = outputSize;
    int sH = stride;
    int kH = kernel;

    if(padding > 0 || (cm == ConvolutionMode.Same && paddingR > 0)){
        //Note: for same mode, bottom/right padding can be 1 more than top/left padding
        //NCW format.
        if(cm == ConvolutionMode.Same) {
            layerInput = sameDiff.nn().pad(layerInput,
                    sameDiff.constant(Nd4j.createFromArray(new int[][]{{0, 0}, {0, 0}, {padding, paddingR}})), PadMode.CONSTANT, 0);
        } else {
            layerInput = sameDiff.nn().pad(layerInput,
                    sameDiff.constant(Nd4j.createFromArray(new int[][]{{0, 0}, {0, 0}, {padding, padding}})), PadMode.CONSTANT, 0);
        }
    }

    SDVariable[] inputArray = new SDVariable[outH];
    for (int i = 0; i < outH; i++) {
        SDVariable slice = layerInput.get(SDIndex.all(), // miniBatch
                        SDIndex.all(), // nIn
                        SDIndex.interval(i * sH, i * sH + kH) // kernel
        );
        inputArray[i] = sameDiff.reshape(slice, 1, -1, featureDim);
    }
    SDVariable concatOutput = sameDiff.concat(0, inputArray); // (outH, miniBatch, featureDim)

    SDVariable mmulResult = sameDiff.mmul(concatOutput, w); // (outH, miniBatch, nOut)

    SDVariable result = sameDiff.permute(mmulResult, 1, 2, 0); // (miniBatch, nOut, outH)

    if (hasBias) {
        SDVariable b = paramTable.get(ConvolutionParamInitializer.BIAS_KEY);
        SDVariable biasAddedResult = sameDiff.nn().biasAdd(result, b, true);
        return activation.asSameDiff("out", sameDiff, biasAddedResult);
    } else {
        return activation.asSameDiff("out", sameDiff, result);
    }

}
 

protected INDArray preOutput(boolean training , LayerWorkspaceMgr workspaceMgr) {

        INDArray bias = getParamWithNoise(DeconvolutionParamInitializer.BIAS_KEY, training, workspaceMgr);
        INDArray weights = getParamWithNoise(DeconvolutionParamInitializer.WEIGHT_KEY, training, workspaceMgr);

        //Input validation: expect rank 5 matrix
        if (input.rank() != 5) {
            throw new DL4JInvalidInputException("Got rank " + input.rank()
                    + " array as input to Deconvolution3DLayer with shape " + Arrays.toString(input.shape())
                    + ". Expected rank 5 array with shape [minibatchSize, channels, inputHeight, inputWidth, inputDepth] or" +
                    " [minibatchSize, inputHeight, inputWidth, inputDepth, channels]. " + layerId());
        }

        Convolution3D.DataFormat df = layerConf().getDataFormat();
        boolean ncdhw = layerConf().getDataFormat() == Convolution3D.DataFormat.NCDHW;
        int chDim = ncdhw ? 1 : 4;
        if (input.size(chDim) != layerConf().getNIn() ) {
            String layerName = conf.getLayer().getLayerName();
            if (layerName == null)
                layerName = "(not named)";
            throw new DL4JInvalidInputException("Cannot do forward pass in Deconvolution3D layer (layer name = " + layerName
                    + ", layer index = " + index + "): input array channels does not match CNN layer configuration"
                    + " (data input channels = " + input.size(chDim) + ", " + (ncdhw ? "[minibatch,channels,height,width,depth]=" : "[minibatch,height,width,depth,channels]=")
                    + Arrays.toString(input.shape()) + "; expected" + " input channels = " + layerConf().getNIn() + ") "
                    + layerId());
        }

        int[] dilation = layerConf().getDilation();
        int[] kernel = layerConf().getKernelSize();
        int[] strides = layerConf().getStride();

        int[] pad;
        ConvolutionMode cm = layerConf().getConvolutionMode();
        long[] outSize;
        int[] inSize = df == Convolution3D.DataFormat.NCDHW ? new int[]{(int)input.size(2), (int)input.size(3), (int)input.size(4)} : new int[]{(int)input.size(1), (int)input.size(2), (int)input.size(3)};
        if (cm == ConvolutionMode.Same) {
            outSize = ConvolutionUtils.getDeconvolution3DOutputSize(input, kernel, strides, null, dilation, cm, layerConf().getDataFormat()); //Also performs validation
            pad = ConvolutionUtils.getSameModeTopLeftPadding(ArrayUtil.toInts(outSize), inSize, kernel, strides, dilation );
        } else {
            pad = layerConf().getPadding();
            outSize = ConvolutionUtils.getDeconvolution3DOutputSize(input, kernel, strides, pad, dilation, cm, layerConf().getDataFormat()); //Also performs validation
        }

        long outH = outSize[0];
        long outW = outSize[1];
        long outD = outSize[2];


        val miniBatch = input.size(0);
        long[] outShape = df == Convolution3D.DataFormat.NCDHW ? new long[]{miniBatch, layerConf().getNOut(), outH, outW, outD} : new long[]{miniBatch, outH, outW, outD, layerConf().getNOut()};
        INDArray output = workspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), outShape, 'c');

        int sameMode = (cm == ConvolutionMode.Same) ? 1 : 0;

        int[] args = new int[] {
                kernel[0], kernel[1], kernel[2], strides[0], strides[1], strides[2],
                pad[0], pad[1], pad[2], dilation[0], dilation[1], dilation[2], sameMode,
                df == Convolution3D.DataFormat.NCDHW ? 0 : 1
        };

        INDArray[] opInputs;
        if (layerConf().hasBias()) {
            opInputs = new INDArray[]{input, weights, bias};
        } else {
            opInputs = new INDArray[]{input, weights};
        }
        CustomOp op = DynamicCustomOp.builder("deconv3d")
                .addInputs(opInputs)
                .addIntegerArguments(args)
                .addOutputs(output)
                .callInplace(false)
                .build();
        Nd4j.getExecutioner().exec(op);

        return output;
    }
 

@Test
public void testGlobalLocalConfigCompGraph() {
    for (ConvolutionMode cm : new ConvolutionMode[] {ConvolutionMode.Strict, ConvolutionMode.Truncate,
                    ConvolutionMode.Same}) {
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
                        .convolutionMode(cm).graphBuilder().addInputs("in")
                        .addLayer("0", new ConvolutionLayer.Builder().kernelSize(3, 3).stride(3, 3).padding(0, 0)
                                        .nIn(3).nOut(
                                                        3)
                                        .build(), "in")
                        .addLayer("1", new ConvolutionLayer.Builder().convolutionMode(ConvolutionMode.Strict)
                                        .kernelSize(3, 3).stride(3, 3).padding(0, 0)
                                        .nIn(3).nOut(
                                                        3)
                                        .build(), "0")
                        .addLayer("2", new ConvolutionLayer.Builder().convolutionMode(ConvolutionMode.Truncate)
                                        .kernelSize(3, 3).stride(3, 3).padding(0, 0)
                                        .nIn(3).nOut(
                                                        3)
                                        .build(), "1")
                        .addLayer("3", new ConvolutionLayer.Builder().convolutionMode(ConvolutionMode.Same)
                                        .kernelSize(3, 3).stride(3, 3).padding(0, 0).nIn(3).nOut(3).build(), "2")
                        .addLayer("4", new SubsamplingLayer.Builder().kernelSize(3, 3).stride(3, 3).padding(0, 0)
                                        .build(), "3")
                        .addLayer("5", new SubsamplingLayer.Builder().convolutionMode(ConvolutionMode.Strict)
                                        .kernelSize(3, 3).stride(3, 3).padding(0, 0).build(), "4")
                        .addLayer("6", new SubsamplingLayer.Builder().convolutionMode(ConvolutionMode.Truncate)
                                        .kernelSize(3, 3).stride(3, 3).padding(0, 0).build(), "5")
                        .addLayer("7", new SubsamplingLayer.Builder().convolutionMode(ConvolutionMode.Same)
                                        .kernelSize(3, 3).stride(3, 3).padding(0, 0).build(), "6")
                        .addLayer("8", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                .activation(Activation.SOFTMAX).nOut(3).build(), "7")
                        .setOutputs("8").build();

        assertEquals(cm, ((ConvolutionLayer) ((LayerVertex) conf.getVertices().get("0")).getLayerConf().getLayer())
                        .getConvolutionMode());
        assertEquals(ConvolutionMode.Strict,
                        ((ConvolutionLayer) ((LayerVertex) conf.getVertices().get("1")).getLayerConf().getLayer())
                                        .getConvolutionMode());
        assertEquals(ConvolutionMode.Truncate,
                        ((ConvolutionLayer) ((LayerVertex) conf.getVertices().get("2")).getLayerConf().getLayer())
                                        .getConvolutionMode());
        assertEquals(ConvolutionMode.Same,
                        ((ConvolutionLayer) ((LayerVertex) conf.getVertices().get("3")).getLayerConf().getLayer())
                                        .getConvolutionMode());

        assertEquals(cm, ((SubsamplingLayer) ((LayerVertex) conf.getVertices().get("4")).getLayerConf().getLayer())
                        .getConvolutionMode());
        assertEquals(ConvolutionMode.Strict,
                        ((SubsamplingLayer) ((LayerVertex) conf.getVertices().get("5")).getLayerConf().getLayer())
                                        .getConvolutionMode());
        assertEquals(ConvolutionMode.Truncate,
                        ((SubsamplingLayer) ((LayerVertex) conf.getVertices().get("6")).getLayerConf().getLayer())
                                        .getConvolutionMode());
        assertEquals(ConvolutionMode.Same,
                        ((SubsamplingLayer) ((LayerVertex) conf.getVertices().get("7")).getLayerConf().getLayer())
                                        .getConvolutionMode());
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    if (input.rank() != 4) {
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                + " array as input to Deconvolution2DLayer with shape " + Arrays.toString(input.shape())
                + ". Expected rank 4 array with shape " + layerConf().getCnn2dDataFormat().dimensionNames() + ". "
                + layerId());
    }

    INDArray weights = getParamWithNoise(DeconvolutionParamInitializer.WEIGHT_KEY, true, workspaceMgr);

    CNN2DFormat format = layerConf().getCnn2dDataFormat();
    boolean nchw = format == CNN2DFormat.NCHW;
    int hDim = nchw ? 2 : 1;
    int wDim = nchw ? 3 : 2;

    long miniBatch = input.size(0);
    long inH = input.size(hDim);
    long inW = input.size(wDim);

    long inDepth = weights.size(0);

    long kH = weights.size(2);
    long kW = weights.size(3);

    int[] dilation = layerConf().getDilation();
    int[] kernel = layerConf().getKernelSize();
    int[] strides = layerConf().getStride();
    int[] pad;
    if (convolutionMode == ConvolutionMode.Same) {
        int[] outSize = new int[]{(int)epsilon.size(hDim), (int)epsilon.size(wDim)};
        pad = ConvolutionUtils.getSameModeTopLeftPadding(outSize, new int[] {(int)inH, (int)inW}, kernel, strides, dilation);
    } else {
        pad = layerConf().getPadding();
    }

    INDArray biasGradView = gradientViews.get(DeconvolutionParamInitializer.BIAS_KEY);
    INDArray weightGradView = gradientViews.get(DeconvolutionParamInitializer.WEIGHT_KEY);

    long[] epsShape = nchw ? new long[]{miniBatch, inDepth, inH, inW} : new long[]{miniBatch, inH, inW, inDepth};
    INDArray outEps = workspaceMgr.create(ArrayType.ACTIVATION_GRAD, weights.dataType(), epsShape, 'c');

    Integer sameMode = (convolutionMode == ConvolutionMode.Same) ? 1 : 0;

    int[] args = new int[] {
            (int)kH, (int)kW, strides[0], strides[1],
            pad[0], pad[1], dilation[0], dilation[1], sameMode,
            nchw ? 0 : 1 //0 = NCHW; 1 = NHWC
    };

    INDArray delta;
    IActivation afn = layerConf().getActivationFn();
    Pair<INDArray, INDArray> p = preOutput4d(true, true, workspaceMgr);
    delta = afn.backprop(p.getFirst(), epsilon).getFirst();

    //DL4J Deconv weights: [inputDepth, outputDepth, kH, kW]
    //libnd4j weights: [kH, kW, oC, iC]
    weights = weights.permute(2, 3, 1, 0);
    INDArray weightGradViewOp = weightGradView.permute(2, 3, 1, 0);

    INDArray[] opInputs;
    INDArray[] opOutputs;
    if(layerConf().hasBias()){
        INDArray bias = getParamWithNoise(DeconvolutionParamInitializer.BIAS_KEY, true, workspaceMgr);
        opInputs = new INDArray[]{input, weights, bias, delta};
        opOutputs = new INDArray[]{outEps, weightGradViewOp, biasGradView};
    } else {
        opInputs = new INDArray[]{input, weights, delta};
        opOutputs = new INDArray[]{outEps, weightGradViewOp};
    }
    CustomOp op = DynamicCustomOp.builder("deconv2d_bp")
            .addInputs(opInputs)
            .addIntegerArguments(args)
            .addOutputs(opOutputs)
            .callInplace(false)
            .build();
    Nd4j.getExecutioner().exec(op);


    Gradient retGradient = new DefaultGradient();
    if(layerConf().hasBias()){
        retGradient.setGradientFor(DeconvolutionParamInitializer.BIAS_KEY, biasGradView);
    }
    retGradient.setGradientFor(DeconvolutionParamInitializer.WEIGHT_KEY, weightGradView, 'c');
    weightNoiseParams.clear();

    return new Pair<>(retGradient, outEps);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    if (training && !dropoutApplied && layerConf().getIDropout() != null) {
        applyDropOutIfNecessary(true, workspaceMgr);
    }

    boolean isNCDHW = layerConf().getDataFormat() == Convolution3D.DataFormat.NCDHW;

    if (input.rank() != 5) {
        if(isNCDHW){
            throw new DL4JInvalidInputException("Got rank " + input.rank()
                    + " array as input to Subsampling3DLayer with shape " + Arrays.toString(input.shape())
                    + ". Expected rank 5 array with shape [minibatchSize, channels, "
                    + "inputDepth, inputHeight, inputWidth] when dataFormat=NCDHW. "
                    + layerId());
        } else {
            throw new DL4JInvalidInputException("Got rank " + input.rank()
                    + " array as input to Subsampling3DLayer with shape " + Arrays.toString(input.shape())
                    + ". Expected rank 5 array with shape [minibatchSize, inputDepth, inputHeight, inputWidth, channels] when dataFormat=NDHWC. "
                    + layerId());
        }
    }

    long miniBatch = input.size(0);
    long inChannels = isNCDHW ? input.size(1) : input.size(4);
    int inD = (int) (isNCDHW ? input.size(2) : input.size(1));
    int inH = (int) (isNCDHW ? input.size(3) : input.size(2));
    int inW = (int) (isNCDHW ? input.size(4) : input.size(3));

    int[] kernel = layerConf().getKernelSize();
    int[] strides = layerConf().getStride();
    int[] dilation = layerConf().getDilation();
    int[] pad;
    int[] outSize;
    if (convolutionMode == ConvolutionMode.Same) {
        int[] inShape = new int[]{inD, inH, inW};
        outSize = Convolution3DUtils.get3DOutputSize(
                input, kernel, strides, null, convolutionMode, dilation, isNCDHW);
        pad = Convolution3DUtils.get3DSameModeTopLeftPadding(outSize, inShape, kernel, strides, dilation);
    } else {
        pad = layerConf().getPadding();
        outSize = Convolution3DUtils.get3DOutputSize(
                input, kernel, strides, pad, convolutionMode, dilation, isNCDHW);
    }
    long outD = outSize[0];
    long outH = outSize[1];
    long outW = outSize[2];

    String opName = layerConf().getPoolingType() == PoolingType.MAX ? "maxpool3dnew" : "avgpool3dnew";

    INDArray output = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.dataType(),
            isNCDHW ? new long[]{miniBatch, inChannels, outD, outH, outW} : new long[]{miniBatch, outD, outH, outW, inChannels}, 'c');

    int[] intArgs = new int[]{
            kernel[0], kernel[1], kernel[2],
            strides[0], strides[1], strides[2],
            pad[0], pad[1], pad[2],
            dilation[0], dilation[1], dilation[2],
            convolutionMode == ConvolutionMode.Same ? 1 : 0,
            0,  //Extra param - 0 = exclude padding for average divisor (only applicable for average pooling)
            isNCDHW ? 0 : 1
    };

    CustomOp op = DynamicCustomOp.builder(opName)
            .addInputs(input)
            .addIntegerArguments(intArgs)
            .addOutputs(output)
            .callInplace(false)
            .build();

    Nd4j.getExecutioner().exec(op);

    return output;
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);

    boolean isNCDHW = layerConf().getDataFormat() == Convolution3D.DataFormat.NCDHW;

    long miniBatch = input.size(0);
    long inChannels = isNCDHW ? input.size(1) : input.size(4);
    int inD = (int) (isNCDHW ? input.size(2) : input.size(1));
    int inH = (int) (isNCDHW ? input.size(3) : input.size(2));
    int inW = (int) (isNCDHW ? input.size(4) : input.size(3));

    int[] kernel = layerConf().getKernelSize();
    int[] strides = layerConf().getStride();
    int[] dilation = layerConf().getDilation();

    int[] pad;
    int[] outSize;
    if (convolutionMode == ConvolutionMode.Same) {
        outSize = Convolution3DUtils.get3DOutputSize(
                input, kernel, strides, null, convolutionMode, dilation, isNCDHW);
        pad = Convolution3DUtils.get3DSameModeTopLeftPadding(
                outSize, new int[]{inD, inH, inW}, kernel, strides, dilation);
    } else {
        pad = layerConf().getPadding();
    }

    INDArray outEpsilon = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, epsilon.dataType(),
            isNCDHW ? new long[]{miniBatch, inChannels, inD, inH, inW} : new long[]{miniBatch, inD, inH, inW, inChannels}, 'c');


    int[] intArgs = new int[]{
            kernel[0], kernel[1], kernel[2],
            strides[0], strides[1], strides[2],
            pad[0], pad[1], pad[2],
            dilation[0], dilation[1], dilation[2],
            convolutionMode == ConvolutionMode.Same ? 1 : 0,
            0,  //Extra param - 0 = exclude padding for average divisor
            isNCDHW ? 0 : 1
    };

    String opName = layerConf().getPoolingType() == PoolingType.MAX ? "maxpool3dnew_bp" : "avgpool3dnew_bp";

    CustomOp op = DynamicCustomOp.builder(opName)
            .addInputs(input, epsilon)
            .addIntegerArguments(intArgs)
            .addOutputs(outEpsilon)
            .callInplace(false)
            .build();

    Nd4j.getExecutioner().exec(op);

    Gradient retGradient = new DefaultGradient();
    outEpsilon = backpropDropOutIfPresent(outEpsilon);
    return new Pair<>(retGradient, outEpsilon);
}
 

@Override
public INDArray preOutput(INDArray input, INDArray weights, INDArray bias, int[] kernel, int[] strides, int[] pad,
                          ConvolutionLayer.AlgoMode mode, ConvolutionLayer.FwdAlgo fwdAlgo, ConvolutionMode convolutionMode,
                          int[] dilation, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) {
    if(input.dataType() != DataType.FLOAT || weights.dataType() != DataType.FLOAT)
        return null;    //MKL-DNN only supports floating point dtype


    int hDim = 2;
    int wDim = 3;
    if(format == CNN2DFormat.NHWC){
        hDim = 1;
        wDim = 2;
    }

    int inH = (int)input.size(hDim);
    int inW = (int)input.size(wDim);
    int[] outSize;
    if (convolutionMode == ConvolutionMode.Same) {
        outSize = ConvolutionUtils.getOutputSize(input, kernel, strides, null, convolutionMode, dilation, format); //Also performs validation
        pad = ConvolutionUtils.getSameModeTopLeftPadding(outSize, new int[] {inH, inW}, kernel, strides, dilation);
    } else {
        outSize = ConvolutionUtils.getOutputSize(input, kernel, strides, pad, convolutionMode, dilation, format); //Also performs validation
    }

    if(context == null ){
        context = Nd4j.getExecutioner().buildContext();
        context.setIArguments(kernel[0], kernel[1],
                strides[0], strides[1],
                pad[0], pad[1],
                dilation[0], dilation[1],
                ArrayUtil.fromBoolean(convolutionMode == ConvolutionMode.Same),
                format == CNN2DFormat.NCHW ? 0 : 1,  //0=NCHW, 1=NHWC
                1   //Weight format: 1 - [oC, iC, kH, kW]
        );
    };

    int outDepth = (int) weights.size(0);
    long[] outShape = (format == CNN2DFormat.NCHW) ? new long[]{input.size(0), outDepth, outSize[0], outSize[1]} : new long[]{input.size(0), outSize[0], outSize[1], outDepth};
    INDArray out = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.dataType(), outShape);

    INDArray[] inputsArr = bias == null ? new INDArray[]{input, weights} : new INDArray[]{input, weights, bias};
    context.purge();
    for( int i=0; i<inputsArr.length; i++ ){
        context.setInputArray(i, inputsArr[i]);
    }

    context.setOutputArray(0, out);
    Conv2D op = new Conv2D();
    Nd4j.exec(op, context);

    return out;
}
 

/**
 * @param poolingType     Used when preparing data for subsampling layers ONLY. Null for convolution layers
 * @return
 */
public static CudnnForwardArgs getCudnnForwardArgs(INDArray input, int[] kernel, int[] strides, int[] padding, int[] dilation,
                                                   ConvolutionMode convolutionMode, PoolingType poolingType, CNN2DFormat format){
    INDArray origInput = input;

    //Check if we need to dup the input: views, non-contiguous, etc. CuDNN also seems to have has issues if strides
    // are non-default for C order - even if they *should* be OK otherwise
    if(input.isView() || !Shape.hasDefaultStridesForShape(input)){
        input = input.dup('c');
    }

    boolean nchw = format == CNN2DFormat.NCHW;
    int hIdx = nchw ? 2 : 1;
    int wIdx = nchw ? 3 : 2;

    val inH = input.size(hIdx);
    val inW = input.size(wIdx);

    boolean manualPadBottom = false;
    boolean manualPadRight = false;

    int[] outSize;
    if (convolutionMode == ConvolutionMode.Same) {
        outSize = ConvolutionUtils.getOutputSize(input, kernel, strides, null, convolutionMode, dilation, format); //Also performs validation
        padding = ConvolutionUtils.getSameModeTopLeftPadding(outSize, new int[] {(int) inH, (int) inW}, kernel, strides, dilation);
        int[] padBottomRight = ConvolutionUtils.getSameModeBottomRightPadding(outSize, new int[] {(int) inH, (int) inW}, kernel, strides, dilation);
        if(!Arrays.equals(padding, padBottomRight)){
            /*
            CuDNN - even as of 7.1 (CUDA 9.1) still doesn't have support for proper SAME mode padding (i.e., asymmetric
            padding) - padding can *only* be specified as the same amount for both the top/bottom, and for left/right.
            In SAME mode padding, sometimes these are the same - but often they are not.
            Note that when they differ, the bottom or right padding will be exactly 1 more than the top or left padding.
            As per TF, we'll manually pad here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/conv_ops.cc#L571-L607
             */
            manualPadBottom = (padding[0] != padBottomRight[0]);
            manualPadRight = (padding[1] != padBottomRight[1]);

            //NCHW format
            long[] newShape;
            if(nchw){
                newShape = new long[]{input.size(0), input.size(1),
                        input.size(2) + (manualPadBottom ? 1 : 0),
                        input.size(3) + (manualPadRight ? 1 : 0)};
            } else {
                newShape = new long[]{input.size(0),
                        input.size(1) + (manualPadBottom ? 1 : 0),
                        input.size(2) + (manualPadRight ? 1 : 0),
                        input.size(3)};
            }
            INDArray newInput;
            if(poolingType == null || poolingType != PoolingType.MAX){
                newInput = Nd4j.create(input.dataType(), newShape);
            } else {
                //For max pooling, we don't want to include the padding in the maximum values. But, CuDNN doesn't knowm
                // that these values are padding and hence should be excluded. Instead: We'll use -infinity so that,
                // if the 'real' (non-padding) values are all < 0, we take the real value, not the padding value
                newInput = Nd4j.valueArrayOf(newShape, Double.NEGATIVE_INFINITY, input.dataType());
            }

            if(nchw){
                newInput.put(new INDArrayIndex[]{all(), all(), interval(0,input.size(2)),
                        interval(0, input.size(3))}, input);
            } else {
                newInput.put(new INDArrayIndex[]{all(), interval(0,input.size(1)),
                        interval(0, input.size(2)), all()}, input);
            }

            input = newInput;
            //Now: we've manually applied the "extra" bottom/right padding only - if required. Consequently, we
            // now have the same amount of padding required for top/bottom, and left/right - which we'll let
            // CuDNN handle
        }
    } else {
        outSize = ConvolutionUtils.getOutputSize(input, kernel, strides, padding, convolutionMode, dilation, format); //Also performs validation
    }

    return new CudnnForwardArgs(manualPadBottom, manualPadRight, input, origInput, padding, outSize);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray input, INDArray epsilon, int[] kernel, int[] strides, int[] pad,
                                                 PoolingType poolingType, ConvolutionMode convolutionMode, int[] dilation,
                                                 CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) {
    if(poolingType == PoolingType.SUM || poolingType == PoolingType.PNORM)
        return null;

    INDArray gradAtInput = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), input.shape());

    int hIdx = 2;
    int wIdx = 3;
    if(format == CNN2DFormat.NHWC){
        hIdx = 1;
        wIdx = 2;
    }

    if (convolutionMode == ConvolutionMode.Same) {
        pad = ConvolutionUtils.getSameModeTopLeftPadding(new int[]{(int)epsilon.size(hIdx), (int)epsilon.size(wIdx)}, new int[] {(int)input.size(hIdx), (int)input.size(wIdx)}, kernel, strides, dilation);
    }

    Pooling2DConfig conf = Pooling2DConfig.builder()
            .isSameMode(convolutionMode == ConvolutionMode.Same)
            .kH(kernel[0]).kW(kernel[1])
            .sH(strides[0]).sW(strides[1])
            .dH(dilation[0]).dW(dilation[1])
            .pH(pad[0]).pW(pad[1])
            .isNHWC(format == CNN2DFormat.NHWC)
            .build();

    switch (poolingType){
        case MAX:
            conf.setType(Pooling2D.Pooling2DType.MAX);
            break;
        case AVG:
            conf.setType(Pooling2D.Pooling2DType.AVG);
            break;
    }

    Pooling2DDerivative d = new Pooling2DDerivative(input, epsilon, gradAtInput, conf);

    Nd4j.exec(d);
    return new Pair<Gradient,INDArray>(new DefaultGradient(), gradAtInput);
}