org.deeplearning4j.nn.api.MaskState Java Examples

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr){
    if (getRnnDataFormat() == RNNFormat.NWC){
        this.input = input.permute(0, 2, 1);
    }
    INDArray act4d = super.activate(training, workspaceMgr);
    INDArray act3d = act4d.reshape(act4d.size(0), act4d.size(1), act4d.size(2));

    if(maskArray != null){
        INDArray maskOut = feedForwardMaskArray(maskArray, MaskState.Active, (int)act3d.size(0)).getFirst();
        Preconditions.checkState(act3d.size(0) == maskOut.size(0) && act3d.size(2) == maskOut.size(1),
                "Activations dimensions (0,2) and mask dimensions (0,1) don't match: Activations %s, Mask %s",
                act3d.shape(), maskOut.shape());
        Broadcast.mul(act3d, maskOut, act3d, 0, 2);
    }
    if (getRnnDataFormat() == RNNFormat.NWC){
        this.input = input.permute(0, 2, 1);
        act3d = act3d.permute(0, 2, 1);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, act3d);   //Should be zero copy most of the time
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    if (maskArrays == null || maskArrays.length == 0) {
        return new Pair<>(null, currentMaskState);
    }

    boolean allNull = true;
    for (int i = 0; i < maskArrays.length; i++) {
        if (maskArrays[i] != null) {
            allNull = false;
            break;
        }
    }
    if (allNull) {
        return new Pair<>(null, currentMaskState);
    }

    //Mask arrays are either 1d (column vector) or 2d...
    long start = from * minibatchSize;
    long end = (from + 1) * minibatchSize;
    INDArray outMask = maskArrays[0].get(NDArrayIndex.interval(start, end), NDArrayIndex.all());
    return new Pair<>(outMask, currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    if(maskArray == null || maskArray.rank() == 2)
        return new Pair<>(maskArray, currentMaskState);

    if (maskArray.rank() != 4 || maskArray.size(2) != 1 || maskArray.size(3) != 1) {
        throw new UnsupportedOperationException(
                "Expected rank 4 mask array for 2D CNN layer activations. Got rank " + maskArray.rank() + " mask array (shape " +
                        Arrays.toString(maskArray.shape()) + ")  - when used in conjunction with input data of shape" +
                        " [batch,channels,h,w] 4d masks passing through CnnToFeedForwardPreProcessor should have shape" +
                        " [batchSize,1,1,1]");
    }

    return new Pair<>(maskArray.reshape(maskArray.ordering(), maskArray.size(0), maskArray.size(1)), currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    if (maskArrays == null || maskArrays.length == 0) {
        return new Pair<>(null, currentMaskState);
    }

    if (layerPreProcessor != null) {
        Pair<INDArray, MaskState> pair =
                        layerPreProcessor.feedForwardMaskArray(maskArrays[0], currentMaskState, minibatchSize);
        if (pair == null) {
            maskArrays[0] = null;
            currentMaskState = null;
        } else {
            maskArrays[0] = pair.getFirst();
            currentMaskState = pair.getSecond();
        }
    }

    return layer.feedForwardMaskArray(maskArrays[0], currentMaskState, minibatchSize);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) {
    if (maskArray == null) {
        //For same mode (with stride 1): output activations size is always same size as input activations size -> mask array is same size
        return new Pair<>(maskArray, currentMaskState);
    }

    INDArray outMask = ConvolutionUtils.cnn2dMaskReduction(maskArray, layerConf().getKernelSize(), layerConf().getStride(),
            layerConf().getPadding(), layerConf().getDilation(), layerConf().getConvolutionMode());
    return new Pair<>(outMask, currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //No op
    if (maskArrays == null || maskArrays.length == 0) {
        return null;
    }

    return new Pair<>(maskArrays[0], currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    //Assume mask array is 4d - a mask array that has been reshaped from [minibatch,timeSeriesLength] to [minibatch*timeSeriesLength, 1, 1, 1]
    if (maskArray == null) {
        return new Pair<>(maskArray, currentMaskState);
    } else {
        //Need to reshape mask array from [minibatch*timeSeriesLength, 1, 1, 1] to [minibatch,timeSeriesLength]
        return new Pair<>(TimeSeriesUtils.reshapeCnnMaskToTimeSeriesMask(maskArray, minibatchSize),currentMaskState);
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                                                      int minibatchSize) {
    //Global pooling layer: no masking is possible after this point... i.e., masks have been taken into account
    // as part of the pooling
    this.maskArray = maskArray;
    this.maskState = null; //Not used in global pooling - always applied

    return null;
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //Input: 2d mask array, for masking a time series. After extracting out the last time step, we no longer need the mask array

    return new Pair<>(null, currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //No op
    if (maskArrays == null || maskArrays.length == 0) {
        return null;
    }

    return new Pair<>(maskArrays[0], currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    //LSTM (standard, not bi-directional) don't make any changes to the data OR the mask arrays
    //Any relevant masking occurs during backprop
    //They also set the current mask array as inactive: this is for situations like the following:
    // in -> dense -> lstm -> dense -> lstm
    // The first dense should be masked using the input array, but the second shouldn't. If necessary, the second
    // dense will be masked via the output layer mask

    return new Pair<>(maskArray, MaskState.Passthrough);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    if(maskArray == null)
        return null;
    this.maskArray = TimeSeriesUtils.reshapeTimeSeriesMaskToVector(maskArray, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT);   //TODO
    this.maskState = currentMaskState;

    return null; //Last layer in network
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) {
    Pair<INDArray,MaskState> ret = fwd.feedForwardMaskArray(maskArray, currentMaskState, minibatchSize);
    bwd.feedForwardMaskArray(TimeSeriesUtils.reverseTimeSeriesMask(maskArray, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT),   //TODO
            currentMaskState, minibatchSize);
    return ret;
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) {
    if(maskArray == null){
        return underlying.feedForwardMaskArray(null, currentMaskState, minibatchSize);
    } else {
        INDArray reshaped = TimeSeriesUtils.reshapeTimeSeriesMaskToVector(maskArray, LayerWorkspaceMgr.noWorkspaces(), ArrayType.ACTIVATIONS);
        Pair<INDArray, MaskState> p = underlying.feedForwardMaskArray(reshaped, currentMaskState, minibatchSize);
        if(p == null || p.getFirst() == null){
            return p;
        }
        INDArray reshaped2 = TimeSeriesUtils.reshapeVectorToTimeSeriesMask(p.getFirst(), (int)maskArray.size(0));
        p.setFirst(reshaped2);
        return p;
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                                                      int minibatchSize) {
    underlying.feedForwardMaskArray(maskArray, currentMaskState, minibatchSize);

    //Input: 2d mask array, for masking a time series. After extracting out the last time step,
    // we no longer need the mask array
    return new Pair<>(null, currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    //LSTM (standard, not bi-directional) don't make any changes to the data OR the mask arrays
    //Any relevant masking occurs during backprop
    //They also set the current mask array as inactive: this is for situations like the following:
    // in -> dense -> lstm -> dense -> lstm
    // The first dense should be masked using the input array, but the second shouldn't. If necessary, the second
    // dense will be masked via the output layer mask

    return new Pair<>(maskArray, MaskState.Passthrough);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) {
    if (maskArray == null) {
        //For same mode (with stride 1): output activations size is always same size as input activations size -> mask array is same size
        return new Pair<>(maskArray, currentMaskState);
    }

    INDArray outMask = ConvolutionUtils.cnn2dMaskReduction(maskArray, layerConf().getKernelSize(), layerConf().getStride(),
            layerConf().getPadding(), layerConf().getDilation(), layerConf().getConvolutionMode());
    return super.feedForwardMaskArray(outMask, currentMaskState, minibatchSize);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    //Assume mask array is 1d - a mask array that has been reshaped from [minibatch,timeSeriesLength] to [minibatch*timeSeriesLength, 1]
    if (maskArray == null) {
        return new Pair<>(maskArray, currentMaskState);
    } else if (maskArray.isVector()) {
        //Need to reshape mask array from [minibatch*timeSeriesLength, 1] to [minibatch,timeSeriesLength]
        return new Pair<>(TimeSeriesUtils.reshapeVectorToTimeSeriesMask(maskArray, minibatchSize),
                        currentMaskState);
    } else {
        throw new IllegalArgumentException("Received mask array with shape " + Arrays.toString(maskArray.shape())
                        + "; expected vector.");
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    if (maskArrays == null) {
        return new Pair<>(null, currentMaskState);
    }

    //Most common case: all or none.
    //If there's only *some* mask arrays: assume the others (missing) are equivalent to all 1s
    //And for handling multiple masks: best strategy seems to be an OR operation
    //i.e., output is 1 if any of the input are 1s
    //Which means: if any masks are missing, output null (equivalent to no mask)
    //Otherwise do an element-wise OR operation

    for (INDArray arr : maskArrays) {
        if (arr == null) {
            return new Pair<>(null, currentMaskState);
        }
    }

    //At this point: all present. Do OR operation
    if (maskArrays.length == 1) {
        return new Pair<>(maskArrays[0], currentMaskState);
    } else {
        INDArray ret;
        if(maskArrays[0].dataType() == DataType.BOOL){
            ret = maskArrays[0].dup(maskArrays[0].ordering());
        } else {
            ret = maskArrays[0].castTo(DataType.BOOL);
        }
        Nd4j.getExecutioner().exec(new Or(ret, maskArrays[1].castTo(DataType.BOOL), ret));
        for (int i = 2; i < maskArrays.length; i++) {
            Nd4j.getExecutioner().exec(new Or(maskArrays[i].castTo(DataType.BOOL), ret, ret));
        }
        return new Pair<>(ret, currentMaskState);
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //No op: subset just provides part of the activations for each example (or time step)
    if (maskArrays == null || maskArrays.length == 0) {
        return null;
    }

    return new Pair<>(maskArrays[0], currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    //Assume mask array is 2d for time series (1 value per time step)
    if (maskArray == null) {
        return new Pair<>(maskArray, currentMaskState);
    } else if (maskArray.rank() == 2) {
        //Need to reshape mask array from [minibatch,timeSeriesLength] to [minibatch*timeSeriesLength, 1]
        return new Pair<>(TimeSeriesUtils.reshapeTimeSeriesMaskToVector(maskArray, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT),  //TODO
                currentMaskState);
    } else {
        throw new IllegalArgumentException("Received mask array of rank " + maskArray.rank()
                        + "; expected rank 2 mask array. Mask array shape: " + Arrays.toString(maskArray.shape()));
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //No op
    if (maskArrays == null || maskArrays.length == 0) {
        return null;
    }

    return new Pair<>(maskArrays[0], currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    for (InputPreProcessor preproc : inputPreProcessors) {
        Pair<INDArray, MaskState> p = preproc.feedForwardMaskArray(maskArray, currentMaskState, minibatchSize);
        maskArray = p.getFirst();
        currentMaskState = p.getSecond();
    }
    return new Pair<>(maskArray, currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState,
                int minibatchSize) {
    //Assume mask array is 2d for time series (1 value per time step)
    if (maskArray == null) {
        return new Pair<>(maskArray, currentMaskState);
    } else if (maskArray.rank() == 2) {
        //Need to reshape mask array from [minibatch,timeSeriesLength] to 4d minibatch format: [minibatch*timeSeriesLength, 1, 1, 1]
        return new Pair<>(TimeSeriesUtils.reshapeTimeSeriesMaskToCnn4dMask(maskArray,
                LayerWorkspaceMgr.noWorkspacesImmutable(), ArrayType.INPUT), currentMaskState);
    } else {
        throw new IllegalArgumentException("Received mask array of rank " + maskArray.rank()
                        + "; expected rank 2 mask array. Mask array shape: " + Arrays.toString(maskArray.shape()));
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //No op
    if (maskArrays == null || maskArrays.length == 0) {
        return null;
    }

    return new Pair<>(maskArrays[0], currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //Present for all time steps, or as per the corresponding input mask (if present)
    INDArray[] allMasks = graph.getInputMaskArrays();
    if (allMasks == null || allMasks[inputVertexIndex] == null) {
        //No mask
        return null;
    }
    return new Pair<>(allMasks[inputVertexIndex], MaskState.Active);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    if (maskArrays == null) {
        return new Pair<>(null, currentMaskState);
    }

    //Most common case: all or none.
    //If there's only *some* mask arrays: assume the others (missing) are equivalent to all 1s
    //And for handling multiple masks: best strategy seems to be an OR operation
    //i.e., output is 1 if any of the input are 1s
    //Which means: if any masks are missing, output null (equivalent to no mask, or all steps present)
    //Otherwise do an element-wise OR operation

    for (INDArray arr : maskArrays) {
        if (arr == null) {
            return new Pair<>(null, currentMaskState);
        }
    }

    //At this point: all present. Do OR operation
    if (maskArrays.length == 1) {
        return new Pair<>(maskArrays[0], currentMaskState);
    } else {
        INDArray ret = maskArrays[0].dup(maskArrays[0].ordering());
        Nd4j.getExecutioner().exec(new Or(maskArrays[0], maskArrays[1], ret));
        for (int i = 2; i < maskArrays.length; i++) {
            Nd4j.getExecutioner().exec(new Or(maskArrays[i], ret, ret));
        }
        return new Pair<>(ret, currentMaskState);
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState, int minibatchSize) {
    if(maskArrays != null) {
        if(maskArrays[0] == null) {
            // Queries are unmasked, we don't need to pass on any mask
            return null;
        }else{
            // Queries are masked, keep the masking going
            return Pair.of(maskArrays[0], currentMaskState);
        }
    }else {
        return Pair.of(null, currentMaskState);
    }
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    //No op
    if (maskArrays == null || maskArrays.length == 0) {
        return null;
    }

    return new Pair<>(maskArrays[0], currentMaskState);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState,
                int minibatchSize) {
    if (maskArrays == null) {
        return new Pair<>(null, currentMaskState);
    }

    //Most common case: all or none.
    //If there's only *some* mask arrays: assume the others (missing) are equivalent to all 1s
    //And for handling multiple masks: best strategy seems to be an OR operation
    //i.e., output is 1 if any of the input are 1s
    //Which means: if any masks are missing, output null (equivalent to no mask, or all steps present)
    //Otherwise do an element-wise OR operation

    for (INDArray arr : maskArrays) {
        if (arr == null) {
            return new Pair<>(null, currentMaskState);
        }
    }

    //At this point: all present. Do OR operation
    if (maskArrays.length == 1) {
        return new Pair<>(maskArrays[0], currentMaskState);
    } else {
        INDArray ret = Nd4j.createUninitialized(DataType.BOOL, maskArrays[0].shape());  //maskArrays[0].dup(maskArrays[0].ordering());
        Nd4j.getExecutioner().exec(new Or(maskArrays[0].castTo(DataType.BOOL), maskArrays[1].castTo(DataType.BOOL), ret));
        for (int i = 2; i < maskArrays.length; i++) {
            Nd4j.getExecutioner().exec(new Or(maskArrays[i].castTo(DataType.BOOL), ret, ret));
        }
        return new Pair<>(ret.castTo(Nd4j.defaultFloatingPointType()), currentMaskState);
    }
}