org.deeplearning4j.exception.DL4JInvalidInputException Java Examples

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    if (input.rank() != 3)
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                + " array as input to Subsampling1DLayer with shape " + Arrays.toString(input.shape())
                + ". Expected rank 3 array with shape [minibatchSize, features, length]. " + layerId());

    // add singleton fourth dimension to input
    INDArray origInput = input;
    input = input.castTo(dataType).reshape(input.size(0), input.size(1), input.size(2), 1);

    // call 2D SubsamplingLayer's activate method
    INDArray acts = super.activate(training, workspaceMgr);

    // remove singleton fourth dimension from input and output activations
    input = origInput;
    acts = acts.reshape(acts.size(0), acts.size(1), acts.size(2));

    return acts;
}
 

protected void validateInputRank() {
    //Input validation: expect rank 4 matrix
    if (input.rank() != 4) {
        String layerName = conf.getLayer().getLayerName();
        if (layerName == null)
            layerName = "(not named)";
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                + " array as input to ConvolutionLayer (layer name = " + layerName + ", layer index = "
                + index + ") with shape " + Arrays.toString(input.shape()) + ". "
                + "Expected rank 4 array with shape [minibatchSize, layerInputDepth, inputHeight, inputWidth]."
                + (input.rank() == 2
                ? " (Wrong input type (see InputType.convolutionalFlat()) or wrong data type?)"
                : "")
                + " " + layerId());
    }
}
 

protected void validateInputDepth(long inDepth) {
    CNN2DFormat format = layerConf().getCnn2dDataFormat();
    int dim = format == CNN2DFormat.NHWC ? 3 : 1;
    if (input.size(dim) != inDepth) {
        String layerName = conf.getLayer().getLayerName();
        if (layerName == null)
            layerName = "(not named)";

        String s = "Cannot do forward pass in Convolution layer (layer name = " + layerName
                + ", layer index = " + index + "): input array channels does not match CNN layer configuration"
                + " (data format = " + format + ", data input channels = " + input.size(dim) + ", " + layerConf().getCnn2dDataFormat().dimensionNames()
                + "=" + Arrays.toString(input.shape()) + "; expected" + " input channels = " + inDepth + ") "
                + layerId();

        int dimIfWrongFormat = format == CNN2DFormat.NHWC ? 1 : 3;
        if(input.size(dimIfWrongFormat) == inDepth){
            //User might have passed NCHW data to a NHWC net, or vice versa?
            s += "\n" + ConvolutionUtils.NCHW_NHWC_ERROR_MSG;
        }


        throw new DL4JInvalidInputException(s);
    }
}
 

@Test
public void testDeconvBadInput(){
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new Deconvolution2D.Builder().nIn(5).nOut(3).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray badInput = Nd4j.create(DataType.FLOAT, 1, 10, 5, 5);
    try {
        net.output(badInput);
    } catch (DL4JInvalidInputException e){
        String msg = e.getMessage();
        assertTrue(msg,msg.contains("Deconvolution2D") && msg.contains("input") && msg.contains("channels"));
    }
}
 

@Override
public void fit(MultiDataSet dataSet) {
    if (dataSet.getFeatures().length == 1 && dataSet.getLabels().length == 1) {
        INDArray features = dataSet.getFeatures(0);
        INDArray labels = dataSet.getLabels(0);
        INDArray fMask = null;
        INDArray lMask = null;

        if (dataSet.getFeaturesMaskArrays() != null)
            fMask = dataSet.getFeaturesMaskArrays()[0];

        if (dataSet.getFeaturesMaskArrays() != null)
            lMask = dataSet.getLabelsMaskArrays()[0];

        DataSet ds = new DataSet(features, labels, fMask, lMask);
        fit(ds);
    } else {
        throw new DL4JInvalidInputException(
                "MultiLayerNetwork can't handle MultiDataSet with more than 1 features or labels array." +
                        "Please consider use of ComputationGraph");
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    if (epsilon.rank() != 3)
        throw new DL4JInvalidInputException("Got rank " + epsilon.rank()
                        + " array as epsilon for Subsampling1DLayer backprop with shape "
                        + Arrays.toString(epsilon.shape())
                        + ". Expected rank 3 array with shape [minibatchSize, features, length]. " + layerId());
    if(maskArray != null){
        INDArray maskOut = feedForwardMaskArray(maskArray, MaskState.Active, (int)epsilon.size(0)).getFirst();
        Preconditions.checkState(epsilon.size(0) == maskOut.size(0) && epsilon.size(2) == maskOut.size(1),
                "Activation gradients dimensions (0,2) and mask dimensions (0,1) don't match: Activation gradients %s, Mask %s",
                epsilon.shape(), maskOut.shape());
        Broadcast.mul(epsilon, maskOut, epsilon, 0, 2);
    }

    // add singleton fourth dimension to input and next layer's epsilon
    INDArray origInput = input;
    input = input.castTo(dataType).reshape(input.size(0), input.size(1), input.size(2), 1);
    epsilon = epsilon.reshape(epsilon.size(0), epsilon.size(1), epsilon.size(2), 1);

    // call 2D SubsamplingLayer's backpropGradient method
    Pair<Gradient, INDArray> gradientEpsNext = super.backpropGradient(epsilon, workspaceMgr);
    INDArray epsNext = gradientEpsNext.getSecond();

    // remove singleton fourth dimension from input and current epsilon
    input = origInput;
    epsNext = epsNext.reshape(epsNext.size(0), epsNext.size(1), epsNext.size(2));

    return new Pair<>(gradientEpsNext.getFirst(), epsNext);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (input.rank() != 3)
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                        + " array as input to Subsampling1DLayer with shape " + Arrays.toString(input.shape())
                        + ". Expected rank 3 array with shape [minibatchSize, features, length]. " + layerId());

    // add singleton fourth dimension to input
    INDArray origInput = input;
    input = input.castTo(dataType).reshape(input.size(0), input.size(1), input.size(2), 1);

    // call 2D SubsamplingLayer's activate method
    INDArray acts = super.activate(training, workspaceMgr);

    // remove singleton fourth dimension from input and output activations
    input = origInput;
    acts = acts.reshape(acts.size(0), acts.size(1), acts.size(2));

    if(maskArray != null){
        INDArray maskOut = feedForwardMaskArray(maskArray, MaskState.Active, (int)acts.size(0)).getFirst();
        Preconditions.checkState(acts.size(0) == maskOut.size(0) && acts.size(2) == maskOut.size(1),
                "Activations dimensions (0,2) and mask dimensions (0,1) don't match: Activations %s, Mask %s",
                acts.shape(), maskOut.shape());
        Broadcast.mul(acts, maskOut, acts, 0, 2);
    }

    return acts;
}
 

@Override
public INDArray preOutput(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    applyDropOutIfNecessary(training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);
    INDArray U = getParamWithNoise(CDAEParameter.USER_KEY, training, workspaceMgr);
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);

    // Input validation:
    if (input.rank() != 2 || input.columns() != W.rows()) {
        if (input.rank() != 2) {
            throw new DL4JInvalidInputException("Input that is not a matrix; expected matrix (rank 2), got rank " + input.rank() + " array with shape " + Arrays.toString(input.shape()) + ". Missing preprocessor or wrong input type? " + layerId());
        }
        throw new DL4JInvalidInputException("Input size (" + input.columns() + " columns; shape = " + Arrays.toString(input.shape()) + ") is invalid: does not match layer input size (layer # inputs = " + W.size(0) + ") " + layerId());
    }

    INDArray ret = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.size(0), W.size(1));
    input.mmuli(W, ret);
    ret.addi(U);
    if (hasBias()) {
        ret.addiRowVector(b);
    }

    if (maskArray != null) {
        applyMask(ret);
    }

    return ret;
}
 

@Override
protected INDArray preOutput(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    if (input.columns() != 1) {
        //Assume shape is [numExamples,1], and each entry is an integer index
        throw new DL4JInvalidInputException(
                        "Cannot do forward pass for embedding layer with input more than one column. "
                                        + "Expected input shape: [numExamples,1] with each entry being an integer index "
                                        + layerId());
    }

    val nIn = layerConf().getNIn();

    if (input.length() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();
    int[] indexes = new int[(int) input.length()];
    for (int i = 0; i < indexes.length; i++) {
        indexes[i] = input.getInt(i, 0);

        if (indexes[i] < 0 || indexes[i] >= nIn) {
            throw new DL4JInvalidInputException("Invalid index for embedding layer: got index " + indexes[i]
                    + " for entry " + i + " in minibatch; indexes must be between 0 and nIn-1 inclusive (0 to "
                    + (nIn  -1) + ")");
        }
    }

    INDArray weights = getParam(DefaultParamInitializer.WEIGHT_KEY);
    INDArray bias = getParam(DefaultParamInitializer.BIAS_KEY);

    INDArray destination = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, weights.dataType(), input.size(0), weights.size(1));
    INDArray rows = Nd4j.pullRows(weights, destination, 1, indexes);
    if(hasBias()){
        rows.addiRowVector(bias);
    }

    return rows;
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    if (input.rank() != 5) {
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                + " array as input to Deconvolution3DLayer with shape " + Arrays.toString(input.shape())
                + ". Expected rank 5 array with shape [minibatchSize, channels, inputHeight, inputWidth, inputDepth] or" +
                " [minibatchSize, inputHeight, inputWidth, inputDepth, channels]. " + layerId());
    }

    INDArray weights = getParamWithNoise(DeconvolutionParamInitializer.WEIGHT_KEY, true, workspaceMgr);

    Convolution3D.DataFormat df = layerConf().getDataFormat();
    ConvolutionMode cm = layerConf().getConvolutionMode();

    int[] dilation = layerConf().getDilation();
    int[] kernel = layerConf().getKernelSize();
    int[] strides = layerConf().getStride();
    int[] pad = layerConf().getPadding();

    INDArray biasGradView = gradientViews.get(DeconvolutionParamInitializer.BIAS_KEY);
    INDArray weightGradView = gradientViews.get(DeconvolutionParamInitializer.WEIGHT_KEY);

    INDArray outEps = workspaceMgr.create(ArrayType.ACTIVATION_GRAD, weights.dataType(), input.shape(), 'c');

    Integer sameMode = (layerConf().getConvolutionMode() == ConvolutionMode.Same) ? 1 : 0;

    int[] args = new int[] {
            kernel[0], kernel[1], kernel[2], strides[0], strides[1], strides[2],
            pad[0], pad[1], pad[2], dilation[0], dilation[1], dilation[2], sameMode,
            df == Convolution3D.DataFormat.NCDHW ? 0 : 1
    };

    INDArray delta;
    IActivation afn = layerConf().getActivationFn();
    INDArray preOutput = preOutput(true, workspaceMgr);
    delta = afn.backprop(preOutput, epsilon).getFirst();

    INDArray[] opInputs;
    INDArray[] opOutputs;
    if(layerConf().hasBias()){
        INDArray bias = getParamWithNoise(DeconvolutionParamInitializer.BIAS_KEY, true, workspaceMgr);
        opInputs = new INDArray[]{input, weights, bias, delta};
        opOutputs = new INDArray[]{outEps, weightGradView, biasGradView};
    } else {
        opInputs = new INDArray[]{input, weights, delta};
        opOutputs = new INDArray[]{outEps, weightGradView};
    }
    CustomOp op = DynamicCustomOp.builder("deconv3d_bp")
            .addInputs(opInputs)
            .addIntegerArguments(args)
            .addOutputs(opOutputs)
            .callInplace(false)
            .build();
    Nd4j.getExecutioner().exec(op);


    Gradient retGradient = new DefaultGradient();
    if(layerConf().hasBias()){
        retGradient.setGradientFor(DeconvolutionParamInitializer.BIAS_KEY, biasGradView);
    }
    retGradient.setGradientFor(DeconvolutionParamInitializer.WEIGHT_KEY, weightGradView, 'c');
    weightNoiseParams.clear();

    return new Pair<>(retGradient, outEps);
}
 

@Override
public Frame<? extends TrainingMessage> frameSequence(Sequence<ShallowSequenceElement> sequence,
                AtomicLong nextRandom, double learningRate) {
    if (vectorsConfiguration.getSampling() > 0)
        sequence = BaseSparkLearningAlgorithm.applySubsampling(sequence, nextRandom, 10L,
                        vectorsConfiguration.getSampling());

    int currentWindow = vectorsConfiguration.getWindow();

    if (vectorsConfiguration.getVariableWindows() != null
                    && vectorsConfiguration.getVariableWindows().length != 0) {
        currentWindow = vectorsConfiguration.getVariableWindows()[RandomUtils
                        .nextInt(0, vectorsConfiguration.getVariableWindows().length)];
    }
    if (frame == null)
        synchronized (this) {
            if (frame == null)
                frame = new ThreadLocal<>();
        }

    if (frame.get() == null)
        frame.set(new Frame<CbowRequestMessage>(BasicSequenceProvider.getInstance().getNextValue()));


    for (int i = 0; i < sequence.getElements().size(); i++) {
        nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
        int b = (int) nextRandom.get() % currentWindow;

        int end = currentWindow * 2 + 1 - b;

        ShallowSequenceElement currentWord = sequence.getElementByIndex(i);

        List<Integer> intsList = new ArrayList<>();
        for (int a = b; a < end; a++) {
            if (a != currentWindow) {
                int c = i - currentWindow + a;
                if (c >= 0 && c < sequence.size()) {
                    ShallowSequenceElement lastWord = sequence.getElementByIndex(c);

                    intsList.add(lastWord.getIndex());
                }
            }
        }

        // basically it's the same as CBOW, we just add labels here
        if (sequence.getSequenceLabels() != null) {
            for (ShallowSequenceElement label : sequence.getSequenceLabels()) {
                intsList.add(label.getIndex());
            }
        } else // FIXME: we probably should throw this exception earlier?
            throw new DL4JInvalidInputException(
                            "Sequence passed via RDD has no labels within, nothing to learn here");


        // just converting values to int
        int[] windowWords = new int[intsList.size()];
        for (int x = 0; x < windowWords.length; x++) {
            windowWords[x] = intsList.get(x);
        }

        if (windowWords.length < 1)
            continue;

        iterateSample(currentWord, windowWords, nextRandom, learningRate, false, 0, true, null);
    }

    Frame<CbowRequestMessage> currentFrame = frame.get();
    frame.set(new Frame<CbowRequestMessage>(BasicSequenceProvider.getInstance().getNextValue()));

    return currentFrame;
}
 

@Override
public void start() throws Exception {
    instance = this;

    String[] pathArr = instanceArgs.ndarrayPath.split(",");
    //INDArray[] pointsArr = new INDArray[pathArr.length];
    // first of all we reading shapes of saved eariler files
    int rows = 0;
    int cols = 0;
    for (int i = 0; i < pathArr.length; i++) {
        DataBuffer shape = BinarySerde.readShapeFromDisk(new File(pathArr[i]));

        log.info("Loading shape {} of {}; Shape: [{} x {}]", i + 1, pathArr.length, Shape.size(shape, 0),
                Shape.size(shape, 1));

        if (Shape.rank(shape) != 2)
            throw new DL4JInvalidInputException("NearestNeighborsServer assumes 2D chunks");

        rows += Shape.size(shape, 0);

        if (cols == 0)
            cols = Shape.size(shape, 1);
        else if (cols != Shape.size(shape, 1))
            throw new DL4JInvalidInputException(
                    "NearestNeighborsServer requires equal 2D chunks. Got columns mismatch.");
    }

    final List<String> labels = new ArrayList<>();
    if (instanceArgs.labelsPath != null) {
        String[] labelsPathArr = instanceArgs.labelsPath.split(",");
        for (int i = 0; i < labelsPathArr.length; i++) {
            labels.addAll(FileUtils.readLines(new File(labelsPathArr[i]), "utf-8"));
        }
    }
    if (!labels.isEmpty() && labels.size() != rows)
        throw new DL4JInvalidInputException(String.format("Number of labels must match number of rows in points matrix (expected %d, found %d)", rows, labels.size()));

    final INDArray points = Nd4j.createUninitialized(rows, cols);

    int lastPosition = 0;
    for (int i = 0; i < pathArr.length; i++) {
        log.info("Loading chunk {} of {}", i + 1, pathArr.length);
        INDArray pointsArr = BinarySerde.readFromDisk(new File(pathArr[i]));

        points.get(NDArrayIndex.interval(lastPosition, lastPosition + pointsArr.rows())).assign(pointsArr);
        lastPosition += pointsArr.rows();

        // let's ensure we don't bring too much stuff in next loop
        System.gc();
    }

    VPTree tree = new VPTree(points, instanceArgs.similarityFunction, instanceArgs.invert);

    //Set play secret key, if required
    //http://www.playframework.com/documentation/latest/ApplicationSecret
    String crypto = System.getProperty("play.crypto.secret");
    if (crypto == null || "changeme".equals(crypto) || "".equals(crypto)) {
        byte[] newCrypto = new byte[1024];

        new Random().nextBytes(newCrypto);

        String base64 = Base64.getEncoder().encodeToString(newCrypto);
        System.setProperty("play.crypto.secret", base64);
    }

    Router r = Router.router(vertx);
    r.route().handler(BodyHandler.create());  //NOTE: Setting this is required to receive request body content at all
    createRoutes(r, labels, tree, points);

    vertx.createHttpServer()
            .requestHandler(r)
            .listen(instanceArgs.port);
}
 

@Test
    public void testWrongFormatIn(){

        for(CNN2DFormat df : CNN2DFormat.values()){


            for(int i=0; i<4; i++ ){

                NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder()
                        .list();
                switch (i){
                    case 0:
                        b.layer(new ConvolutionLayer.Builder().kernelSize(2,2).nIn(3).nOut(3).dataFormat(df).build());
                        break;
                    case 1:
                        b.layer(new DepthwiseConvolution2D.Builder().kernelSize(2,2).nIn(3).nOut(3).dataFormat(df).build());
                        break;
                    case 2:
                        b.layer(new Deconvolution2D.Builder().dataFormat(df).kernelSize(2,2).nIn(3).nOut(3).build());
                        break;
                    case 3:
                        b.layer(new SeparableConvolution2D.Builder().dataFormat(df).kernelSize(2,2).nIn(3).nOut(3).build());
                        break;
                }

                MultiLayerNetwork net = new MultiLayerNetwork(b.build());
                net.init();

                INDArray in;
                INDArray wrongFormatIn;
                if(df == CNN2DFormat.NCHW){
                    in = Nd4j.create(DataType.FLOAT, 5, 3, 12, 12);
                    wrongFormatIn = Nd4j.create(DataType.FLOAT, 5, 12, 12, 3);
                } else {
                    in = Nd4j.create(DataType.FLOAT, 5, 12, 12, 3);
                    wrongFormatIn = Nd4j.create(DataType.FLOAT, 5, 3, 12, 12);
                }

                net.output(in);

                try {
                    net.output(wrongFormatIn);
                } catch (DL4JInvalidInputException e){
//                    e.printStackTrace();
                    String msg = e.getMessage();
                    assertTrue(msg, msg.contains(ConvolutionUtils.NCHW_NHWC_ERROR_MSG));
                }
            }
        }


    }
 

protected Pair<INDArray, INDArray> preOutputWithPreNorm(boolean training, boolean forBackprop, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(forBackprop);
    applyDropOutIfNecessary(training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);
    INDArray g = (hasLayerNorm() ? getParam(DefaultParamInitializer.GAIN_KEY) : null);

    INDArray input = this.input.castTo(dataType);

    //Input validation:
    if (input.rank() != 2 || input.columns() != W.rows()) {
        if (input.rank() != 2) {
            throw new DL4JInvalidInputException("Input that is not a matrix; expected matrix (rank 2), got rank "
                    + input.rank() + " array with shape " + Arrays.toString(input.shape())
                    + ". Missing preprocessor or wrong input type? " + layerId());
        }
        throw new DL4JInvalidInputException(
                "Input size (" + input.columns() + " columns; shape = " + Arrays.toString(input.shape())
                        + ") is invalid: does not match layer input size (layer # inputs = "
                        + W.size(0) + ") " + layerId());
    }


    INDArray ret = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, W.dataType(), input.size(0), W.size(1));
    input.castTo(ret.dataType()).mmuli(W, ret);     //TODO Can we avoid this cast? (It sohuld be a no op if not required, however)

    INDArray preNorm = ret;
    if(hasLayerNorm()){
        preNorm = (forBackprop ? ret.dup(ret.ordering()) : ret);
        Nd4j.getExecutioner().exec(new LayerNorm(preNorm, g, ret, true, 1));
    }

    if(hasBias()){
        ret.addiRowVector(b);
    }

    if (maskArray != null) {
        applyMask(ret);
    }

    return new Pair<>(ret, preNorm);
}
 

protected INDArray preOutput(boolean training, boolean forBackprop, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    applyDropOutIfNecessary(training, null);

    if (input.rank() != 4) {
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                + " array as input to space to batch with shape " + Arrays.toString(input.shape())
                + ". Expected rank 4 array with shape " + layerConf().getFormat().dimensionNames() + ". "
                + layerId());
    }

    if (preOutput != null && forBackprop) {
        return preOutput;
    }

    boolean nchw = layerConf().getFormat() == CNN2DFormat.NCHW;

    long inMiniBatch = input.size(0);
    long depth = input.size(nchw ? 1 : 3);
    long inH = input.size(nchw ? 2 : 1);
    long inW = input.size(nchw ? 3 : 2);

    int[] blocks = getBlocks();
    int[][] padding = getPadding();

    long paddedH = inH + padding[0][0] + padding[0][1];
    long paddedW = inW + padding[1][0] + padding[1][1];

    long outH = paddedH / blocks[0];
    long outW = paddedW / blocks[1];
    long outMiniBatch = inMiniBatch * blocks[0] * blocks[1];

    long[] outShape = nchw ? new long[]{outMiniBatch, depth, outH, outW} : new long[]{outMiniBatch, outH, outW, depth};

    INDArray out = workspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), outShape, 'c');

    INDArray inNHWC = nchw ? input.permute(0, 2, 3, 1) : input;
    INDArray outNHWC = nchw ? out.permute(0, 2, 3, 1) : out;

    CustomOp op = DynamicCustomOp.builder("space_to_batch_nd")
            .addInputs(inNHWC, getBlocksArray(), getPaddingArray())
            .addOutputs(outNHWC)
            .build();
    Nd4j.exec(op);

    return out;
}
 

protected INDArray preOutput(boolean training , LayerWorkspaceMgr workspaceMgr) {

        INDArray bias = getParamWithNoise(DeconvolutionParamInitializer.BIAS_KEY, training, workspaceMgr);
        INDArray weights = getParamWithNoise(DeconvolutionParamInitializer.WEIGHT_KEY, training, workspaceMgr);

        //Input validation: expect rank 5 matrix
        if (input.rank() != 5) {
            throw new DL4JInvalidInputException("Got rank " + input.rank()
                    + " array as input to Deconvolution3DLayer with shape " + Arrays.toString(input.shape())
                    + ". Expected rank 5 array with shape [minibatchSize, channels, inputHeight, inputWidth, inputDepth] or" +
                    " [minibatchSize, inputHeight, inputWidth, inputDepth, channels]. " + layerId());
        }

        Convolution3D.DataFormat df = layerConf().getDataFormat();
        boolean ncdhw = layerConf().getDataFormat() == Convolution3D.DataFormat.NCDHW;
        int chDim = ncdhw ? 1 : 4;
        if (input.size(chDim) != layerConf().getNIn() ) {
            String layerName = conf.getLayer().getLayerName();
            if (layerName == null)
                layerName = "(not named)";
            throw new DL4JInvalidInputException("Cannot do forward pass in Deconvolution3D layer (layer name = " + layerName
                    + ", layer index = " + index + "): input array channels does not match CNN layer configuration"
                    + " (data input channels = " + input.size(chDim) + ", " + (ncdhw ? "[minibatch,channels,height,width,depth]=" : "[minibatch,height,width,depth,channels]=")
                    + Arrays.toString(input.shape()) + "; expected" + " input channels = " + layerConf().getNIn() + ") "
                    + layerId());
        }

        int[] dilation = layerConf().getDilation();
        int[] kernel = layerConf().getKernelSize();
        int[] strides = layerConf().getStride();

        int[] pad;
        ConvolutionMode cm = layerConf().getConvolutionMode();
        long[] outSize;
        int[] inSize = df == Convolution3D.DataFormat.NCDHW ? new int[]{(int)input.size(2), (int)input.size(3), (int)input.size(4)} : new int[]{(int)input.size(1), (int)input.size(2), (int)input.size(3)};
        if (cm == ConvolutionMode.Same) {
            outSize = ConvolutionUtils.getDeconvolution3DOutputSize(input, kernel, strides, null, dilation, cm, layerConf().getDataFormat()); //Also performs validation
            pad = ConvolutionUtils.getSameModeTopLeftPadding(ArrayUtil.toInts(outSize), inSize, kernel, strides, dilation );
        } else {
            pad = layerConf().getPadding();
            outSize = ConvolutionUtils.getDeconvolution3DOutputSize(input, kernel, strides, pad, dilation, cm, layerConf().getDataFormat()); //Also performs validation
        }

        long outH = outSize[0];
        long outW = outSize[1];
        long outD = outSize[2];


        val miniBatch = input.size(0);
        long[] outShape = df == Convolution3D.DataFormat.NCDHW ? new long[]{miniBatch, layerConf().getNOut(), outH, outW, outD} : new long[]{miniBatch, outH, outW, outD, layerConf().getNOut()};
        INDArray output = workspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), outShape, 'c');

        int sameMode = (cm == ConvolutionMode.Same) ? 1 : 0;

        int[] args = new int[] {
                kernel[0], kernel[1], kernel[2], strides[0], strides[1], strides[2],
                pad[0], pad[1], pad[2], dilation[0], dilation[1], dilation[2], sameMode,
                df == Convolution3D.DataFormat.NCDHW ? 0 : 1
        };

        INDArray[] opInputs;
        if (layerConf().hasBias()) {
            opInputs = new INDArray[]{input, weights, bias};
        } else {
            opInputs = new INDArray[]{input, weights};
        }
        CustomOp op = DynamicCustomOp.builder("deconv3d")
                .addInputs(opInputs)
                .addIntegerArguments(args)
                .addOutputs(output)
                .callInplace(false)
                .build();
        Nd4j.getExecutioner().exec(op);

        return output;
    }
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    if (input.rank() != 4) {
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                + " array as input to Deconvolution2DLayer with shape " + Arrays.toString(input.shape())
                + ". Expected rank 4 array with shape " + layerConf().getCnn2dDataFormat().dimensionNames() + ". "
                + layerId());
    }

    INDArray weights = getParamWithNoise(DeconvolutionParamInitializer.WEIGHT_KEY, true, workspaceMgr);

    CNN2DFormat format = layerConf().getCnn2dDataFormat();
    boolean nchw = format == CNN2DFormat.NCHW;
    int hDim = nchw ? 2 : 1;
    int wDim = nchw ? 3 : 2;

    long miniBatch = input.size(0);
    long inH = input.size(hDim);
    long inW = input.size(wDim);

    long inDepth = weights.size(0);

    long kH = weights.size(2);
    long kW = weights.size(3);

    int[] dilation = layerConf().getDilation();
    int[] kernel = layerConf().getKernelSize();
    int[] strides = layerConf().getStride();
    int[] pad;
    if (convolutionMode == ConvolutionMode.Same) {
        int[] outSize = new int[]{(int)epsilon.size(hDim), (int)epsilon.size(wDim)};
        pad = ConvolutionUtils.getSameModeTopLeftPadding(outSize, new int[] {(int)inH, (int)inW}, kernel, strides, dilation);
    } else {
        pad = layerConf().getPadding();
    }

    INDArray biasGradView = gradientViews.get(DeconvolutionParamInitializer.BIAS_KEY);
    INDArray weightGradView = gradientViews.get(DeconvolutionParamInitializer.WEIGHT_KEY);

    long[] epsShape = nchw ? new long[]{miniBatch, inDepth, inH, inW} : new long[]{miniBatch, inH, inW, inDepth};
    INDArray outEps = workspaceMgr.create(ArrayType.ACTIVATION_GRAD, weights.dataType(), epsShape, 'c');

    Integer sameMode = (convolutionMode == ConvolutionMode.Same) ? 1 : 0;

    int[] args = new int[] {
            (int)kH, (int)kW, strides[0], strides[1],
            pad[0], pad[1], dilation[0], dilation[1], sameMode,
            nchw ? 0 : 1 //0 = NCHW; 1 = NHWC
    };

    INDArray delta;
    IActivation afn = layerConf().getActivationFn();
    Pair<INDArray, INDArray> p = preOutput4d(true, true, workspaceMgr);
    delta = afn.backprop(p.getFirst(), epsilon).getFirst();

    //DL4J Deconv weights: [inputDepth, outputDepth, kH, kW]
    //libnd4j weights: [kH, kW, oC, iC]
    weights = weights.permute(2, 3, 1, 0);
    INDArray weightGradViewOp = weightGradView.permute(2, 3, 1, 0);

    INDArray[] opInputs;
    INDArray[] opOutputs;
    if(layerConf().hasBias()){
        INDArray bias = getParamWithNoise(DeconvolutionParamInitializer.BIAS_KEY, true, workspaceMgr);
        opInputs = new INDArray[]{input, weights, bias, delta};
        opOutputs = new INDArray[]{outEps, weightGradViewOp, biasGradView};
    } else {
        opInputs = new INDArray[]{input, weights, delta};
        opOutputs = new INDArray[]{outEps, weightGradViewOp};
    }
    CustomOp op = DynamicCustomOp.builder("deconv2d_bp")
            .addInputs(opInputs)
            .addIntegerArguments(args)
            .addOutputs(opOutputs)
            .callInplace(false)
            .build();
    Nd4j.getExecutioner().exec(op);


    Gradient retGradient = new DefaultGradient();
    if(layerConf().hasBias()){
        retGradient.setGradientFor(DeconvolutionParamInitializer.BIAS_KEY, biasGradView);
    }
    retGradient.setGradientFor(DeconvolutionParamInitializer.WEIGHT_KEY, weightGradView, 'c');
    weightNoiseParams.clear();

    return new Pair<>(retGradient, outEps);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    if (training && !dropoutApplied && layerConf().getIDropout() != null) {
        applyDropOutIfNecessary(true, workspaceMgr);
    }

    boolean isNCDHW = layerConf().getDataFormat() == Convolution3D.DataFormat.NCDHW;

    if (input.rank() != 5) {
        if(isNCDHW){
            throw new DL4JInvalidInputException("Got rank " + input.rank()
                    + " array as input to Subsampling3DLayer with shape " + Arrays.toString(input.shape())
                    + ". Expected rank 5 array with shape [minibatchSize, channels, "
                    + "inputDepth, inputHeight, inputWidth] when dataFormat=NCDHW. "
                    + layerId());
        } else {
            throw new DL4JInvalidInputException("Got rank " + input.rank()
                    + " array as input to Subsampling3DLayer with shape " + Arrays.toString(input.shape())
                    + ". Expected rank 5 array with shape [minibatchSize, inputDepth, inputHeight, inputWidth, channels] when dataFormat=NDHWC. "
                    + layerId());
        }
    }

    long miniBatch = input.size(0);
    long inChannels = isNCDHW ? input.size(1) : input.size(4);
    int inD = (int) (isNCDHW ? input.size(2) : input.size(1));
    int inH = (int) (isNCDHW ? input.size(3) : input.size(2));
    int inW = (int) (isNCDHW ? input.size(4) : input.size(3));

    int[] kernel = layerConf().getKernelSize();
    int[] strides = layerConf().getStride();
    int[] dilation = layerConf().getDilation();
    int[] pad;
    int[] outSize;
    if (convolutionMode == ConvolutionMode.Same) {
        int[] inShape = new int[]{inD, inH, inW};
        outSize = Convolution3DUtils.get3DOutputSize(
                input, kernel, strides, null, convolutionMode, dilation, isNCDHW);
        pad = Convolution3DUtils.get3DSameModeTopLeftPadding(outSize, inShape, kernel, strides, dilation);
    } else {
        pad = layerConf().getPadding();
        outSize = Convolution3DUtils.get3DOutputSize(
                input, kernel, strides, pad, convolutionMode, dilation, isNCDHW);
    }
    long outD = outSize[0];
    long outH = outSize[1];
    long outW = outSize[2];

    String opName = layerConf().getPoolingType() == PoolingType.MAX ? "maxpool3dnew" : "avgpool3dnew";

    INDArray output = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.dataType(),
            isNCDHW ? new long[]{miniBatch, inChannels, outD, outH, outW} : new long[]{miniBatch, outD, outH, outW, inChannels}, 'c');

    int[] intArgs = new int[]{
            kernel[0], kernel[1], kernel[2],
            strides[0], strides[1], strides[2],
            pad[0], pad[1], pad[2],
            dilation[0], dilation[1], dilation[2],
            convolutionMode == ConvolutionMode.Same ? 1 : 0,
            0,  //Extra param - 0 = exclude padding for average divisor (only applicable for average pooling)
            isNCDHW ? 0 : 1
    };

    CustomOp op = DynamicCustomOp.builder(opName)
            .addInputs(input)
            .addIntegerArguments(intArgs)
            .addOutputs(output)
            .callInplace(false)
            .build();

    Nd4j.getExecutioner().exec(op);

    return output;
}
 

/** Returns tuple: {Gradient,Delta,Output} given preOut */
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    if (labels2d.size(1) != preOut.size(1)) {
        throw new DL4JInvalidInputException(
                        "Labels array numColumns (size(1) = " + labels2d.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOut.size(1) + ") " + layerId());
    }

    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(CenterLossParamInitializer.WEIGHT_KEY);
    INDArray biasGradView = gradientViews.get(CenterLossParamInitializer.BIAS_KEY);
    INDArray centersGradView = gradientViews.get(CenterLossParamInitializer.CENTER_KEY);

    // centers delta
    double alpha = layerConf().getAlpha();

    INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
    INDArray l = labels.castTo(centers.dataType()); //Ensure correct dtype (same as params); no-op if already correct dtype
    INDArray centersForExamples = l.mmul(centers);
    INDArray diff = centersForExamples.sub(input).muli(alpha);
    INDArray numerator = l.transpose().mmul(diff);
    INDArray denominator = l.sum(0).reshape(l.size(1), 1).addi(1.0);

    INDArray deltaC;
    if (layerConf().getGradientCheck()) {
        double lambda = layerConf().getLambda();
        //For gradient checks: need to multiply dLc/dcj by lambda to get dL/dcj
        deltaC = numerator.muli(lambda);
    } else {
        deltaC = numerator.diviColumnVector(denominator);
    }
    centersGradView.assign(deltaC);



    // other standard calculations
    Nd4j.gemm(input, delta, weightGradView, true, false, 1.0, 0.0); //Equivalent to:  weightGradView.assign(input.transpose().mmul(delta));
    delta.sum(biasGradView, 0); //biasGradView is initialized/zeroed first in sum op

    gradient.gradientForVariable().put(CenterLossParamInitializer.WEIGHT_KEY, weightGradView);
    gradient.gradientForVariable().put(CenterLossParamInitializer.BIAS_KEY, biasGradView);
    gradient.gradientForVariable().put(CenterLossParamInitializer.CENTER_KEY, centersGradView);

    return new Pair<>(gradient, delta);
}