Java Code Examples for org.deeplearning4j.nn.workspace.LayerWorkspaceMgr#dup()

@Override
public INDArray doForward(boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoForward())
        throw new IllegalStateException("Cannot do forward pass: input not set");

    forwardShape = Arrays.copyOf(inputs[0].shape(), inputs[0].rank());

    INDArray out;
    switch (inputs[0].rank()) {
        case 2:
            out = inputs[0].get(NDArrayIndex.all(), NDArrayIndex.interval(from, to, true));
            break;
        case 3:
            out = inputs[0].get(NDArrayIndex.all(), NDArrayIndex.interval(from, to, true), NDArrayIndex.all());
            break;
        case 4:
            out = inputs[0].get(NDArrayIndex.all(), NDArrayIndex.interval(from, to, true), NDArrayIndex.all(),
                            NDArrayIndex.all());
            break;
        default:
            throw new UnsupportedOperationException(
                            "Cannot get subset for activations of rank " + inputs[0].rank());
    }
    return workspaceMgr.dup(ArrayType.ACTIVATIONS, out);
}
 

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoBackward()) {
        throw new IllegalStateException("Cannot do backward pass: errors not set");
    }
    // epsilons[index] => {batchSize, numberOfEmbeds}
    INDArray[] epsilons = new INDArray[inputs.length];
    // epsilon => {batchSize, 1}
    // inputs[index] => {batchSize, numberOfEmbeds}
    // TODO 如何通过inputs[index]与epsilon求导epsilons[index]
    INDArray output = doForward(true, workspaceMgr);
    for (int index = 0; index < inputs.length; index++) {
        epsilons[index] = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, inputs[index]);
        epsilons[index].muliColumnVector(epsilon).diviColumnVector(output);
    }
    return new Pair<>(null, epsilons);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    if (input.rank() != 4)
        throw new UnsupportedOperationException(
                "Input must be rank 4 with shape " + layerConf().getFormat().dimensionNames() +
                        ". Got input with rank " + input.rank() + " " + layerId());

    CNN2DFormat format = layerConf().getFormat();

    INDArray in = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, input.ordering());
    INDArray input2d = ConvolutionUtils.reshape4dTo2d(in, format, workspaceMgr, ArrayType.ACTIVATIONS);
    INDArray out2d = layerConf().getActivationFn().getActivation(input2d, training);

    return ConvolutionUtils.reshape2dTo4d(out2d, input.shape(), format, workspaceMgr, ArrayType.ACTIVATIONS);
}
 

public static INDArray reshape2dTo4d(INDArray in2d, long[] toShape, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(in2d.rank() != 2)
        throw new IllegalArgumentException("Invalid input: expect NDArray with rank 2");
    if (toShape.length != 4)
        throw new IllegalArgumentException("Invalid input: expect toShape with 4 elements: got " + Arrays.toString(toShape));

    if (in2d.ordering() != 'c' || !Shape.hasDefaultStridesForShape(in2d))
        in2d = workspaceMgr.dup(type, in2d, 'c');

    if(format == CNN2DFormat.NCHW) {
        //Reshape: from [n*h*w,c] to [n,h,w,c] to [n,c,h,w]
        INDArray out = in2d.reshape('c', toShape[0], toShape[2], toShape[3], toShape[1]);
        return workspaceMgr.leverageTo(type, out.permute(0, 3, 1, 2));
    } else {
        //Reshape: from [n*h*w,c] to [n,h,w,c]
        return workspaceMgr.leverageTo(type, in2d.reshape('c', toShape));
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    INDArray layerInput = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, input, input.ordering());

    INDArray alpha = getParam(PReLUParamInitializer.WEIGHT_KEY);
    IActivation prelu = new ActivationPReLU(alpha, axes);

    Pair<INDArray, INDArray> deltas = prelu.backprop(layerInput, epsilon);
    INDArray delta = deltas.getFirst();
    INDArray weightGrad = deltas.getSecond();
    INDArray weightGradView = gradientViews.get(PReLUParamInitializer.WEIGHT_KEY);
    weightGradView.assign(weightGrad);


    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);  //Usually a no-op (except for perhaps identity)
    delta = backpropDropOutIfPresent(delta);
    Gradient ret = new DefaultGradient();
    ret.setGradientFor(PReLUParamInitializer.WEIGHT_KEY, weightGradView, 'c');

    return new Pair<>(ret, delta);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    // the target shape read from a keras config does not have mini-batch size included. We prepend it here dynamically.
    long[] targetShape = getShape(this.targetShape, miniBatchSize);
    long[] inputShape = getShape(this.inputShape, miniBatchSize);

    if (prodLong(input.shape()) == prodLong((targetShape))) {
        if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input)) {
            input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');
        }
        return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.reshape(targetShape));
    } else {
        throw new IllegalStateException("Input shape " + Arrays.toString(input.shape())
                + " and output shape" + Arrays.toString(inputShape) + " do not match");
    }
}
 

/**
 * Reshape time series mask arrays. This should match the assumptions (f order, etc) in RnnOutputLayer
 * This reshapes from [X,1] to [X,1,1,1] suitable for per-example masking in CNNs
 * @param timeSeriesMask    Mask array to reshape for CNN per-example masking
 * @return                  Mask array as 4D CNN mask array: [X, 1, 1, 1]
 */
public static INDArray reshapeTimeSeriesMaskToCnn4dMask(INDArray timeSeriesMask, LayerWorkspaceMgr workspaceMgr, ArrayType arrayType) {
    if (timeSeriesMask.rank() != 2)
        throw new IllegalArgumentException("Cannot reshape mask: rank is not 2");

    if (timeSeriesMask.ordering() != 'f' || !Shape.hasDefaultStridesForShape(timeSeriesMask))
        timeSeriesMask = workspaceMgr.dup(arrayType, timeSeriesMask, 'f');

    return workspaceMgr.leverageTo(arrayType, timeSeriesMask.reshape('f', timeSeriesMask.length(), 1, 1, 1));
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    if (input.rank() != 5)
        throw new UnsupportedOperationException(
                "Input must be rank 5. Got input with rank " + input.rank() + " " + layerId());

    INDArray in = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, input.ordering());
    INDArray input2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), in, workspaceMgr, ArrayType.ACTIVATIONS);
    INDArray out2d = layerConf().getActivationFn().getActivation(input2d, training);

    long n = input.size(0);
    long d, h, w, c;
    if(layerConf().getDataFormat() == Convolution3D.DataFormat.NDHWC){
        d = (int)input.size(1);
        h = (int)input.size(2);
        w = (int)input.size(3);
        c = (int)input.size(4);
    } else {
        d = (int)input.size(2);
        h = (int)input.size(3);
        w = (int)input.size(4);
        c = (int)input.size(1);
    }

    return ConvolutionUtils.reshape2dTo5d(layerConf().getDataFormat(), out2d, n, d, h, w, c, workspaceMgr, ArrayType.ACTIVATIONS);
}
 

@Override
public INDArray doForward(boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoForward())
        throw new IllegalStateException("Cannot do forward pass: inputs not set");

    if (inputs.length > 1)
        throw new IllegalStateException("PoolHelper vertex requires a single input.");

    INDArray strippedInput = inputs[0].get(NDArrayIndex.all(), NDArrayIndex.all(),
                    NDArrayIndex.interval(1, inputs[0].size(2)), NDArrayIndex.interval(1, inputs[0].size(3)));
    return workspaceMgr.dup(ArrayType.ACTIVATIONS, strippedInput);
}
 

/**
 * Reshape time series mask arrays. This should match the assumptions (f order, etc) in RnnOutputLayer
 * @param timeSeriesMask    Mask array to reshape to a column vector
 * @return                  Mask array as a column vector
 */
public static INDArray reshapeTimeSeriesMaskToVector(INDArray timeSeriesMask, LayerWorkspaceMgr workspaceMgr, ArrayType arrayType) {
    if (timeSeriesMask.rank() != 2)
        throw new IllegalArgumentException("Cannot reshape mask: rank is not 2");

    if (timeSeriesMask.ordering() != 'f' || !Shape.hasDefaultStridesForShape(timeSeriesMask))
        timeSeriesMask = workspaceMgr.dup(arrayType, timeSeriesMask, 'f');

    return workspaceMgr.leverageTo(arrayType, timeSeriesMask.reshape('f', timeSeriesMask.length(), 1));
}
 

public static INDArray reshape2dTo3d(INDArray in, long miniBatchSize, LayerWorkspaceMgr workspaceMgr, ArrayType arrayType) {
    if (in.rank() != 2)
        throw new IllegalArgumentException("Invalid input: expect NDArray with rank 2");
    //Based on: RnnToFeedForwardPreProcessor
    val shape = in.shape();
    if (in.ordering() != 'f') {
        in = workspaceMgr.dup(arrayType, in, 'f');
    }
    INDArray reshaped = in.reshape('f', miniBatchSize, shape[0] / miniBatchSize, shape[1]);
    return workspaceMgr.leverageTo(arrayType, reshaped.permute(0, 2, 1));
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    INDArray reshapedEps = reshape(epsilon);
    Pair<Gradient, INDArray> p = underlying.backpropGradient(reshapedEps, workspaceMgr);
    INDArray reverted = revertReshape(p.getSecond(), epsilon.size(0));
    reverted = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, reverted);
    p.setSecond(reverted);
    return p;
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (output.ordering() != 'c' || !Shape.hasDefaultStridesForShape(output)) {
        output = workspaceMgr.dup(ArrayType.ACTIVATIONS, output, 'c');
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output.permute(permutationIndices));

}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (permutationIndices.length + 1 == input.shape().length) {
        permutationIndices = prependZero(permutationIndices);
        this.hasLeadingDimension = true;
    }
    if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input)) {
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');
    }
    INDArray output = workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.permute(this.permutationIndices));
    return output;
}
 

@Override
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (epsilons.ordering() != 'c' || !Shape.hasDefaultStridesForShape(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    INDArray epsilonsReshaped = epsilons.reshape('c', epsilons.size(0), inputHeight, inputWidth, numChannels);

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilonsReshaped.permute(0, 3, 1, 2));    //To [n, c, h, w]
}
 

public static INDArray reshape2dTo5d(Convolution3D.DataFormat format, INDArray in2d, long n, long d, long h, long w, long ch, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(in2d.rank() != 2)
        throw new IllegalArgumentException("Invalid input: expect NDArray with rank 2");

    //Reshape: from [n*d*h*w,c] to [n,d,h,w,c]; if NCDHW format permute to [n,c,d,h,w]
    if(in2d.ordering() != 'c' || !Shape.hasDefaultStridesForShape(in2d))
        in2d = workspaceMgr.dup(type, in2d, 'c');

    INDArray ndhwc = in2d.reshape('c', n, d, h, w, ch);
    if(format == Convolution3D.DataFormat.NDHWC){
        return workspaceMgr.leverageTo(type, ndhwc);
    } else {
        return workspaceMgr.leverageTo(type, ndhwc.permute(0, 4, 1, 2, 3));
    }
}
 

@Override
// return 4 dimensions
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (epsilons.ordering() != 'c' || !Shape.hasDefaultStridesForShape(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    if (shape == null || ArrayUtil.prod(shape) != epsilons.length()) {
        if (epsilons.rank() == 2)
            return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons); //should never happen

        return epsilons.reshape('c', epsilons.size(0), numChannels, inputHeight, inputWidth);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons.reshape('c', shape));
}
 

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {
    // this is basically doForward on UnstackVertex
    if (!canDoForward())
        throw new IllegalStateException("Cannot do forward pass: input not set");

    if (epsilon == null) {
        //Edge case for stack vertex: stack -> embedding
        //If the null epsilons are a problem in practice, this should be picked up by other layers
        return new Pair<>(null, new INDArray[inputs.length]);
    }

    int nStack = inputs.length;
    INDArray[] out = new INDArray[nStack];

    long step = epsilon.size(0) / nStack;

    for (int i = 0; i < nStack; i++) {
        switch (epsilon.rank()) {
            case 2:
                out[i] = epsilon.get(NDArrayIndex.interval(i * step, (i + 1) * step), NDArrayIndex.all());
                break;
            case 3:
                if (lastInputShapes != null) {
                    //Variable length time series case
                    out[i] = epsilon.get(NDArrayIndex.interval(i * step, (i + 1) * step), NDArrayIndex.all(),
                                    NDArrayIndex.interval(0, lastInputShapes[i][2]));
                } else {
                    out[i] = epsilon.get(NDArrayIndex.interval(i * step, (i + 1) * step), NDArrayIndex.all(),
                                    NDArrayIndex.all());
                }
                break;
            case 4:
                out[i] = epsilon.get(NDArrayIndex.interval(i * step, (i + 1) * step), NDArrayIndex.all(),
                                NDArrayIndex.all(), NDArrayIndex.all());
                break;
            default:
                throw new UnsupportedOperationException(
                                "Cannot get subset for activations of rank " + inputs[0].rank());
        }
    }

    for( int i=0; i<nStack; i++ ){
        out[i] = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, out[i]);
    }

    return new Pair<>(null, out);
}
 

private INDArray activateHelper(boolean activations, LayerWorkspaceMgr workspaceMgr){
    assertInputSet(false);

    //Check where the output occurs. If it's a simple loss layer (no params) this could
    // just be the input!
    if(activations && INPUT_KEY.equals(layerConf().activationsVertexName())){
        return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input);
    }

    //TODO optimize
    try(MemoryWorkspace ws = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
        if (sameDiff == null) {
            doInit();
        }
    }

    //Configure memory management for SameDiff instance - use DL4J workspaces
    String wsNameWorking = workspaceMgr.getWorkspaceName(ArrayType.FF_WORKING_MEM);
    String wsNameOutput = workspaceMgr.getWorkspaceName(ArrayType.ACTIVATIONS);
    WorkspaceConfiguration confWorking = workspaceMgr.getConfiguration(ArrayType.FF_WORKING_MEM);
    WorkspaceConfiguration confOutput = workspaceMgr.getConfiguration(ArrayType.ACTIVATIONS);
    boolean actScopedOut = workspaceMgr.isScopedOut(ArrayType.ACTIVATIONS);
    Preconditions.checkState(actScopedOut || wsNameOutput != null, "Activations must have a workspace or must be scoped out");
    SessionMemMgr mmgr = new DL4JSameDiffMemoryMgr(wsNameWorking, wsNameOutput, confWorking, confOutput);

    InferenceSession is = sameDiff.getSessions().get(Thread.currentThread().getId());
    if(is == null){
        is = new InferenceSession(sameDiff);
        sameDiff.getSessions().put(Thread.currentThread().getId(), is);
    }
    is.setMmgr(mmgr);

    Map<String,INDArray> phMap = new HashMap<>();
    phMap.put(INPUT_KEY, input);
    if(!activations && layerConf().labelsRequired() && labels != null) {
        phMap.put(LABELS_KEY, labels);
    }

    String s = activations ? layerConf().activationsVertexName() : outputVar.name();

    INDArray out = sameDiff.outputSingle(phMap, s);

    //Clear placeholders and op inputs to ensure no out-of-scope arrays are still referenced anywhere
    sameDiff.clearPlaceholders(true);
    sameDiff.clearOpInputs();

    //Edge case: vertex is just an Identity function, for example
    //TODO there may be a cleaner way to do this...
    if(!actScopedOut && !out.data().getParentWorkspace().getId().equals(wsNameOutput)){
        out = workspaceMgr.dup(ArrayType.ACTIVATIONS, out);
    } else if(actScopedOut && out.isAttached()){
        out = out.detach();
    }

    return out;
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr mgr) {
    return mgr.dup(ArrayType.ACTIVATIONS, input);
}