Java Code Examples for org.deeplearning4j.nn.workspace.LayerWorkspaceMgr#leverageTo()

@Override
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Epsilons from layer above should be 2d, with shape [miniBatchSize, depthOut*outH*outW]
    if (epsilons.ordering() != 'c' || !Shape.strideDescendingCAscendingF(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    if (epsilons.rank() == 4)
        return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons); //Should never happen

    if (epsilons.columns() != inputWidth * inputHeight * numChannels)
        throw new IllegalArgumentException("Invalid input: expect output columns must be equal to rows "
                        + inputHeight + " x columns " + inputWidth + " x channels " + numChannels + " but was instead "
                        + Arrays.toString(epsilons.shape()));

    INDArray ret;
    if(format == CNN2DFormat.NCHW){
        ret = epsilons.reshape('c', epsilons.size(0), numChannels, inputHeight, inputWidth);
    } else {
        ret = epsilons.reshape('c', epsilons.size(0), inputHeight, inputWidth, numChannels);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, ret); //Move if required to specified workspace
}
 

/** Returns tuple: {Gradient,Delta,Output} given preOut */
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    //INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFunction(), maskArray);
    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.gemm(input.castTo(weightGradView.dataType()), delta, weightGradView, true, false, 1.0, 0.0); //Equivalent to:  weightGradView.assign(input.transpose().mmul(delta));         //TODO can we avoid cast?
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradView);

    if(hasBias()){
        INDArray biasGradView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGradView, 0); //biasGradView is initialized/zeroed first in sum op
        gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradView);
    }

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Need to reshape RNN activations (3d) activations to 2d (for input into feed forward layer)
    if (input.rank() != 3)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 3 (i.e., activations for RNN layer)");

    if (input.ordering() != 'f' || !Shape.hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'f');

    if (rnnDataFormat == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
    }
    val shape = input.shape();
    INDArray ret;
    if (shape[0] == 1) {
        ret = input.tensorAlongDimension(0, 1, 2).permute(1, 0); //Edge case: miniBatchSize==1
    } else if (shape[2] == 1) {
        ret = input.tensorAlongDimension(0, 1, 0); //Edge case: timeSeriesLength=1
    } else {
        INDArray permuted = input.permute(0, 2, 1); //Permute, so we get correct order after reshaping
        ret = permuted.reshape('f', shape[0] * shape[2], shape[1]);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, ret);
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = preOutput2d(false, workspaceMgr);

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
            lossFunction.computeScoreArray(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM),
                    preOut, layerConf().getActivationFn(), maskArray);
    if (fullNetRegTerm != 0.0) {
        scoreArray.addi(fullNetRegTerm);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, scoreArray);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Need to reshape FF activations (2d) activations to 3d (for input into RNN layer)
    if (input.rank() != 2)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 2 (i.e., activations for FF layer)");
    if (input.ordering() != 'f' || !Shape.hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'f');

    val shape = input.shape();
    INDArray reshaped = input.reshape('f', miniBatchSize, shape[0] / miniBatchSize, shape[1]);
    if (rnnDataFormat == RNNFormat.NCW){
        reshaped = reshaped.permute(0, 2, 1);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, reshaped);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    // the target shape read from a keras config does not have mini-batch size included. We prepend it here dynamically.
    long[] targetShape = getShape(this.targetShape, miniBatchSize);
    long[] inputShape = getShape(this.inputShape, miniBatchSize);

    if (prodLong(input.shape()) == prodLong((targetShape))) {
        if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input)) {
            input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');
        }
        return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.reshape(targetShape));
    } else {
        throw new IllegalStateException("Input shape " + Arrays.toString(input.shape())
                + " and output shape" + Arrays.toString(inputShape) + " do not match");
    }
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (input.rank() == 2)
        return input; // Pass-through feed-forward input

    // We expect either NCDHW or NDHWC format
    if ((isNCDHW && input.size(1) != numChannels) || (!isNCDHW && input.size(4) != numChannels)) {
        throw new IllegalStateException("Invalid input array: expected shape in format "
                + "[minibatch, channels, channels, height, width] or "
                + "[minibatch, channels, height, width, channels]"
                + " for numChannels: " + numChannels + ", inputDepth " + inputDepth + ", inputHeight " + inputHeight
                + " and inputWidth " + inputWidth + ", but got "
                + Arrays.toString(input.shape()));
    }

    if (!hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');

    val inShape = input.shape();
    val outShape = new long[]{inShape[0], inShape[1] * inShape[2] * inShape[3] * inShape[4]};

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.reshape('c', outShape));
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    if (input.rank() != 5)
        throw new UnsupportedOperationException(
                "Input is not rank 5. Got input with rank " + input.rank() + " " + layerId() + " with shape "
                        + Arrays.toString(input.shape()) + " - expected shape [minibatch,channels,depth,height,width]");
    if (labels == null)
        throw new IllegalStateException("Labels are not set (null)");

    INDArray input2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), input, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray labels2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), labels, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray maskReshaped = ConvolutionUtils.reshapeCnn3dMask(layerConf().getDataFormat(), maskArray, labels, workspaceMgr, ArrayType.FF_WORKING_MEM);

    // delta calculation
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray delta2d = lossFunction.computeGradient(labels2d, input2d.dup(input2d.ordering()), layerConf().getActivationFn(), maskReshaped);
    delta2d = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta2d);

    long n = input.size(0);
    long d, h, w, c;
    if(layerConf().getDataFormat() == Convolution3D.DataFormat.NDHWC){
        d = input.size(1);
        h = input.size(2);
        w = input.size(3);
        c = input.size(4);
    } else {
        d = input.size(2);
        h = input.size(3);
        w = input.size(4);
        c = input.size(1);
    }
    INDArray delta5d = ConvolutionUtils.reshape2dTo5d(layerConf().getDataFormat(), delta2d, n, d, h, w, c, workspaceMgr, ArrayType.ACTIVATION_GRAD);

    // grab the empty gradient
    Gradient gradient = new DefaultGradient();
    return new Pair<>(gradient, delta5d);
}
 

@Override
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (!hasDefaultStridesForShape(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    if (shape == null || ArrayUtil.prod(shape) != epsilons.length()) {
        INDArray ret = epsilons.reshape('c', epsilons.size(0),inputDepth * inputHeight * inputWidth * numChannels);
        return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, ret);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons.reshape('c', shape));
}
 

public static INDArray reshape5dTo2d(@NonNull Convolution3D.DataFormat format, INDArray in, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    Preconditions.checkState(in.rank() == 5, "Invalid input: expect NDArray with rank 5, got rank %ndRank with shape %ndShape", in, in);
    //Reshape: from either [n,c,d,h,w] to [n*d*h*w,c] (NCDHW format)
    // or reshape from [n,d,h,w,c] to [n*d*h*w,c] (NDHWC format)
    if(format != Convolution3D.DataFormat.NDHWC){
        in = in.permute(0, 2, 3, 4, 1);
    }

    if(in.ordering() != 'c' || !Shape.hasDefaultStridesForShape(in))
        in = workspaceMgr.dup(type, in, 'c');
    return workspaceMgr.leverageTo(type, in.reshape('c', in.size(0)*in.size(1)*in.size(2)*in.size(3), in.size(4)));
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (input.rank() == 2)
        return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input); //Should usually never happen
    /* DL4J convolutional input:       # channels, # rows, # cols
     * TensorFlow convolutional input: # rows, # cols, # channels
     * Theano convolutional input:     # channels, # rows, # cols
     */
    INDArray permuted = workspaceMgr.dup(ArrayType.ACTIVATIONS, input.permute(0, 2, 3, 1), 'c'); //To: [n, h, w, c]

    val inShape = input.shape(); //[miniBatch,depthOut,outH,outW]
    val outShape = new long[]{inShape[0], inShape[1] * inShape[2] * inShape[3]};

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, permuted.reshape('c', outShape));
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = input;

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
                    lossFunction.computeScoreArray(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray);
    if (fullNetRegTerm != 0.0) {
        scoreArray.addi(fullNetRegTerm);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, scoreArray);
}
 

/** Returns tuple: {Gradient,Delta,Output} given preOut */
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    // delta calculation
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray delta = lossFunction.computeGradient(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray);

    // grab the empty gradient
    Gradient gradient = new DefaultGradient();

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray z = input;
    INDArray ret = layerConf().getActivationFn().getActivation(z.dup(), training);

    if (maskArray != null) {
        ret.muliColumnVector(maskArray);
    }

    INDArray out = workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, ret);
    return out;
}
 

/**
 * Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    //For 3D CNN: need to sum up the score over each x/y/z location before returning

    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());

    INDArray input2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), input, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray labels2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), labels, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray maskReshaped = ConvolutionUtils.reshapeCnn3dMask(layerConf().getDataFormat(), maskArray, input, workspaceMgr, ArrayType.FF_WORKING_MEM);

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
            lossFunction.computeScoreArray(labels2d, input2d, layerConf().getActivationFn(), maskReshaped);
    //scoreArray: shape [minibatch*d*h*w, 1]
    //Reshape it to [minibatch, 1, d, h, w] then sum over x/y/z to give [minibatch, 1]

    val newShape = input.shape().clone();
    newShape[1] = 1;

    long n = input.size(0);
    long d, h, w, c;
    if(layerConf().getDataFormat() == Convolution3D.DataFormat.NDHWC){
        d = input.size(1);
        h = input.size(2);
        w = input.size(3);
        c = input.size(4);
    } else {
        d = input.size(2);
        h = input.size(3);
        w = input.size(4);
        c = input.size(1);
    }
    INDArray scoreArrayTs = ConvolutionUtils.reshape2dTo5d(layerConf().getDataFormat(), scoreArray, n, d, h, w, c, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray summedScores = scoreArrayTs.sum(1,2,3,4);

    if (fullNetRegTerm != 0.0) {
        summedScores.addi(fullNetRegTerm);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, summedScores);
}
 

private static INDArray applyMask(INDArray input, INDArray maskArray, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(maskArray == null){
        return workspaceMgr.leverageTo(type, input);
    }
    switch (input.rank()){
        case 2:
            if(!maskArray.isColumnVectorOrScalar() || maskArray.size(0) != input.size(0)){
                throw new IllegalStateException("Expected column vector for mask with 2d input, with same size(0)" +
                        " as input. Got mask with shape: " + Arrays.toString(maskArray.shape()) +
                        ", input shape = " + Arrays.toString(input.shape()));
            }
            return workspaceMgr.leverageTo(type, input.mulColumnVector(maskArray));
        case 3:
            //Time series input, shape [Minibatch, size, tsLength], Expect rank 2 mask
            if(maskArray.rank() != 2 || input.size(0) != maskArray.size(0) || input.size(2) != maskArray.size(1)){
                throw new IllegalStateException("With 3d (time series) input with shape [minibatch, size, sequenceLength]=" +
                        Arrays.toString(input.shape()) + ", expected 2d mask array with shape [minibatch, sequenceLength]." +
                        " Got mask with shape: "+ Arrays.toString(maskArray.shape()));
            }
            INDArray fwd = workspaceMgr.createUninitialized(type, input.dataType(), input.shape(), 'f');
            Broadcast.mul(input, maskArray, fwd, 0, 2);
            return fwd;
        case 4:
            //CNN input. Expect column vector to be shape [mb,1,h,1], [mb,1,1,w], or [mb,1,h,w]
            int[] dimensions = new int[4];
            int count = 0;
            for(int i=0; i<4; i++ ){
                if(input.size(i) == maskArray.size(i)){
                    dimensions[count++] = i;
                }
            }
            if(count < 4){
                dimensions = Arrays.copyOfRange(dimensions, 0, count);
            }

            INDArray fwd2 = workspaceMgr.createUninitialized(type, input.dataType(), input.shape(), 'c');
            Broadcast.mul(input, maskArray, fwd2, dimensions);
            return fwd2;
        default:
            throw new RuntimeException("Expected rank 2 to 4 input. Got rank " + input.rank() + " with shape "
                    + Arrays.toString(input.shape()));
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);

    if (!layerConf().isCollapseDimensions() && epsilon.rank() != 2) {
        val origShape = epsilon.shape();
        //Don't collapse dims case: error should be [minibatch, vectorSize, 1] or [minibatch, channels, 1, 1]
        //Reshape it to 2d, to get rid of the 1s
        epsilon = epsilon.reshape(epsilon.ordering(), origShape[0], origShape[1]);
    }

    INDArray input = this.input.castTo(dataType);       //No-op if already correct dtype

    Gradient retGradient = new DefaultGradient(); //Empty: no params

    int[] poolDim = null;
    if (input.rank() == 3) {
        if (poolingDimensions == null) {
            //Use default pooling dimensions;
            poolDim = DEFAULT_TIMESERIES_POOL_DIMS;
        } else {
            poolDim = poolingDimensions;
        }

    } else if (input.rank() == 4) {
        //CNN activations
        if (poolingDimensions == null) {
            //Use default pooling dimensions;
            poolDim = DEFAULT_CNN_POOL_DIMS;
        } else {
            poolDim = poolingDimensions;
        }
    } else if (input.rank() == 5) {
        //CNN activations
        if (poolingDimensions == null) {
            //Use default pooling dimensions;
            poolDim = DEFAULT_CNN3D_POOL_DIMS;
        } else {
            poolDim = poolingDimensions;
        }
    }

    // TODO: masking for CNN3D case
    INDArray epsilonNd;
    if (maskArray == null) {
        //Standard 'full array' global pooling op
        epsilonNd = epsilonHelperFullArray(input, epsilon, poolDim);
    } else {
        if (input.rank() == 3) {
            epsilonNd = MaskedReductionUtil.maskedPoolingEpsilonTimeSeries(poolingType, input, maskArray, epsilon,
                    pNorm);
        } else if (input.rank() == 4) {
            epsilonNd = MaskedReductionUtil.maskedPoolingEpsilonCnn(poolingType, input, maskArray, epsilon, pNorm, dataType);
        } else {
            throw new UnsupportedOperationException(layerId());
        }

    }

    //TODO optimize without leverage
    epsilonNd = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilonNd);
    return new Pair<>(retGradient, epsilonNd);
}
 

@Override
// return 2 dimensions
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (input.rank() == 2)
        return input; //Should usually never happen

    int chDim = 1;
    int hDim = 2;
    int wDim = 3;
    if(format == CNN2DFormat.NHWC){
        chDim = 3;
        hDim = 1;
        wDim = 2;
    }

    if(inputHeight == 0 && inputWidth == 0 && numChannels == 0){
        this.inputHeight = input.size(hDim);
        this.inputWidth = input.size(wDim);
        this.numChannels = input.size(chDim);
    }

    if(input.size(chDim) != numChannels || input.size(hDim) != inputHeight || input.size(wDim) != inputWidth){
        throw new IllegalStateException("Invalid input, does not match configuration: expected " +
                (format == CNN2DFormat.NCHW ? "[minibatch, numChannels=" + numChannels + ", inputHeight=" + inputHeight + ", inputWidth=" + inputWidth + "] " :
                        "[minibatch, inputHeight=" + inputHeight + ", inputWidth=" + inputWidth + ", numChannels=" + numChannels + "]") +
                        " but got input array of shape " + Arrays.toString(input.shape()));
    }

    //Check input: nchw format
    if(input.size(chDim) != numChannels || input.size(hDim) != inputHeight ||
            input.size(wDim) != inputWidth){
        throw new IllegalStateException("Invalid input array: expected shape [minibatch, channels, height, width] = "
                + "[minibatch, " + numChannels + ", " + inputHeight + ", " + inputWidth + "] - got "
                + Arrays.toString(input.shape()));
    }

    //Assume input is standard rank 4 activations out of CNN layer
    //First: we require input to be in c order. But c order (as declared in array order) isn't enough; also need strides to be correct
    if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');

    //Note that to match Tensorflow/Keras, we do a simple "c order reshape" for both NCHW and NHWC

    val inShape = input.shape(); //[miniBatch,depthOut,outH,outW]
    val outShape = new long[]{inShape[0], inShape[1] * inShape[2] * inShape[3]};

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.reshape('c', outShape));    //Should be zero copy reshape
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    for (InputPreProcessor preProcessor : inputPreProcessors)
        input = preProcessor.preProcess(input, miniBatchSize, workspaceMgr);
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input);
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output);
}