Java Code Examples for org.deeplearning4j.nn.workspace.LayerWorkspaceMgr#notifyScopeEntered()

The following examples show how to use org.deeplearning4j.nn.workspace.LayerWorkspaceMgr#notifyScopeEntered() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MultiLayerNetwork.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/**
 * Perform layerwise unsupervised training on a single pre-trainable layer in the network (VAEs, Autoencoders, etc)<br>
 * If the specified layer index (0 to numLayers - 1) is not a pretrainable layer, this is a no-op.
 *
 * @param layerIdx Index of the layer to train (0 to numLayers-1)
 * @param features Training data array
 */
public void pretrainLayer(int layerIdx, INDArray features) {
    setInput(features);
    setLayerMaskArrays(null, null);

    if (flattenedGradients == null) {
        initGradientsView();
    }
    if (layerIdx >= layers.length) {
        throw new IllegalArgumentException(
                "Cannot pretrain layer: layerIdx (" + layerIdx + ") >= numLayers (" + layers.length + ")");
    }

    LayerWorkspaceMgr workspaceMgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE){
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .defaultWorkspace(WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    Layer layer = layers[layerIdx];
    if (!layer.isPretrainLayer())
        return;

    //Do forward pass to the layer to be pretrained
    INDArray outputOfPrevLayer;
    if(layerIdx == 0) {
        outputOfPrevLayer = input;
    } else {
        //Yes, this part of training - but we'll do forward psas as inference mode when doing layerwise training
        // to effectively freeze earlier layers and not apply dropout etc
        outputOfPrevLayer = outputOfLayerDetached(false, FwdPassType.STANDARD, layerIndex-1, features, null, null, null);
    }

    try(MemoryWorkspace ws = workspaceMgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)) {
        if (layerWiseConfigurations.getInputPreProcess(layerIdx) != null) {

            if (input.size(0) > Integer.MAX_VALUE)
                throw new ND4JArraySizeException();
            outputOfPrevLayer = layerWiseConfigurations.getInputPreProcess(layerIdx).preProcess(outputOfPrevLayer, (int) input.size(0),
                    LayerWorkspaceMgr.noWorkspaces(helperWorkspaces));
        }

        layer.fit(outputOfPrevLayer, workspaceMgr);
    }
}
 
Example 2
Source File: MultiLayerNetwork.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/**
 * Feed-forward through the network - returning all array activations in a list, detached from any workspace.
 * Note that no workspace should be active externally when calling this method (an exception will be thrown
 * if a workspace is open externally)
 *
 * @param train             Training mode (true) or test/inference mode (false)
 * @param fwdPassType       Type of forward pass to perform (STANDARD or RNN_ACTIVATE_WITH_STORED_STATE only)
 * @param storeLastForTBPTT ONLY used if fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE
 * @param layerIndex        Index (inclusive) to stop forward pass at. For all layers, use numLayers-1
 * @param input             Input to the network
 * @param fMask             Feature mask array. May be null.
 * @param lMask             Label mask array. May be null.
 * @param clearInputs       Whether the layer inputs should be cleared
 * @return List of activations (including the input), detached from any workspace
 */
protected synchronized List<INDArray> ffToLayerActivationsDetached(boolean train, @NonNull FwdPassType fwdPassType,
                                                      boolean storeLastForTBPTT, int layerIndex, @NonNull INDArray input,
                                                      INDArray fMask, INDArray lMask, boolean clearInputs){
    setInput(input);
    setLayerMaskArrays(fMask, lMask);

    //Verify that no workspace is open externally
    WorkspaceUtils.assertNoWorkspacesOpen("Expected no workspace active in ffToLayerActivationsDetached");

    LayerWorkspaceMgr workspaceMgr;
    WorkspaceMode wsm = (train ? layerWiseConfigurations.getTrainingWorkspaceMode() : layerWiseConfigurations.getInferenceWorkspaceMode());
    if(wsm == WorkspaceMode.NONE){
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .noWorkspaceFor(ArrayType.ACTIVATIONS)
                .with(ArrayType.INPUT, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();

        if(input.isAttached()){
            //Don't leverage out of async DataSetIterator workspaces
            workspaceMgr.setNoLeverageOverride(input.data().getParentWorkspace().getId());
        }

        if(!clearInputs){
            workspaceMgr.setScopedOutFor(ArrayType.INPUT);
        }
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    List<INDArray> out = new ArrayList<>();
    out.add(workspaceMgr.leverageTo(ArrayType.INPUT, input));    //Should  be unnecessary (and no op), if layer is implemented correctly

    for( int i=0; i<=layerIndex; i++ ){
        try(MemoryWorkspace wsFFWorking = workspaceMgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)){
            if (getLayerWiseConfigurations().getInputPreProcess(i) != null) {
                input = getLayerWiseConfigurations().getInputPreProcess(i).preProcess(input, getInputMiniBatchSize(), workspaceMgr);
                //Validation: Exception if invalid (bad preprocessor implementation)
                validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, true, "Feed forward to layer (inference)");
            }

            if(fwdPassType == FwdPassType.STANDARD){
                input = layers[i].activate(input, train, workspaceMgr);
            } else if (fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE) {
                if (layers[i] instanceof RecurrentLayer) {
                    input = ((RecurrentLayer) layers[i]).rnnActivateUsingStoredState(input, train,
                            storeLastForTBPTT, workspaceMgr);
                } else if(layers[i] instanceof BaseWrapperLayer && ((BaseWrapperLayer)layers[i]).getUnderlying() instanceof RecurrentLayer) {
                    RecurrentLayer rl = (RecurrentLayer) ((BaseWrapperLayer)layers[i]).getUnderlying();
                    input = rl.rnnActivateUsingStoredState(input, train,storeLastForTBPTT, workspaceMgr);
                } else if (layers[i] instanceof MultiLayerNetwork) {
                    List<INDArray> temp = ((MultiLayerNetwork) layers[i]).rnnActivateUsingStoredState(input, train, storeLastForTBPTT);
                    input = temp.get(temp.size() - 1);
                } else {
                    input = layers[i].activate(input, train, workspaceMgr);
                }
            } else {
                throw new IllegalStateException("Forward pass type not supported for this method: " + fwdPassType);
            }

            //Validation: Exception if invalid (bad layer implementation)
            validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, false, "Feed forward to layer (inference)");

            out.add(input);
        }
        if(clearInputs) {
            layers[i].clear();
        }
    }

    return out;
}
 
Example 3
Source File: MultiLayerNetwork.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/**
 * Feed-forward through the network at training time - returning a list of all activations in a workspace (WS_ALL_LAYERS_ACT)
 * if workspaces are enabled for training; or detached if no workspaces are used.<br>
 * Note: if using workspaces for training, this method requires that WS_ALL_LAYERS_ACT is open externally.<br>
 * If using NO workspaces, requires that no external workspace is open<br>
 * Note that this method does NOT clear the inputs to each layer - instead, they are in the WS_ALL_LAYERS_ACT workspace
 * for use in later backprop.
 *
 * @param layerIndex        Index (inclusive) to stop forward pass at. For all layers, use numLayers-1
 * @param fwdPassType       Type of forward pass to perform (STANDARD or RNN_ACTIVATE_WITH_STORED_STATE only)
 * @param storeLastForTBPTT ONLY used if fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE
 * @param input             Input to network
 * @param fMask             Feature mask array. May be null
 * @param lMask             Label mask aray. May be null.
 * @return
 */
protected synchronized List<INDArray> ffToLayerActivationsInWs(int layerIndex, @NonNull FwdPassType fwdPassType, boolean storeLastForTBPTT,
                                                  @NonNull INDArray input, INDArray fMask, INDArray lMask){
    setInput(input);
    setLayerMaskArrays(fMask, lMask);

    LayerWorkspaceMgr workspaceMgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE){
        WorkspaceUtils.assertNoWorkspacesOpen("Expected no workspace active in ffToLayerActivationsInWs when training workspace is set to NONE");
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.INPUT, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.ACTIVATIONS, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();

        if(input.isAttached()){
            //Don't leverage out of async DataSetIterator workspaces
            workspaceMgr.setNoLeverageOverride(input.data().getParentWorkspace().getId());
        }

        if(layerWiseConfigurations.getCacheMode() != CacheMode.NONE){
            //For now: store cache mode activations in activations workspace
            workspaceMgr.setWorkspace(ArrayType.FF_CACHE, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG);
            workspaceMgr.setWorkspace(ArrayType.BP_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG);
        }

        WorkspaceUtils.assertOpenAndActive(WS_ALL_LAYERS_ACT, "ffToLayerActivationsInWs method requires workspace WS_ALL_LAYERS_ACT to be open");
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    List<INDArray> out = new ArrayList<>();
    out.add(workspaceMgr.leverageTo(ArrayType.INPUT, input));    //Probably unnecessary usually

    boolean traceLog = log.isTraceEnabled();

    for( int i=0; i<=layerIndex; i++ ){
        try(MemoryWorkspace wsFFWorking = workspaceMgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)){
            if (getLayerWiseConfigurations().getInputPreProcess(i) != null) {
                input = getLayerWiseConfigurations().getInputPreProcess(i).preProcess(input, getInputMiniBatchSize(), workspaceMgr);
                //Validation: Exception if invalid (bad preprocessor implementation)
                validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, true, "Feed forward to layer (training)");
            }

            if(traceLog){
                log.trace("About to forward pass: {} - {}", i, layers[i].getClass().getSimpleName());
            }

            if(fwdPassType == FwdPassType.STANDARD){
                input = layers[i].activate(input, true, workspaceMgr);
            } else if(fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE){
                if (layers[i] instanceof RecurrentLayer) {
                    input = ((RecurrentLayer) layers[i]).rnnActivateUsingStoredState(input, true, storeLastForTBPTT, workspaceMgr);
                }else if(layers[i] instanceof BaseWrapperLayer && ((BaseWrapperLayer)layers[i]).getUnderlying() instanceof RecurrentLayer) {
                    RecurrentLayer rl = (RecurrentLayer) ((BaseWrapperLayer)layers[i]).getUnderlying();
                    input = rl.rnnActivateUsingStoredState(input, true, storeLastForTBPTT, workspaceMgr);
                } else if (layers[i] instanceof MultiLayerNetwork) {
                    List<INDArray> temp = ((MultiLayerNetwork) layers[i]).rnnActivateUsingStoredState(input, true, storeLastForTBPTT);
                    input = temp.get(temp.size() - 1);
                } else {
                    input = layers[i].activate(input, true, workspaceMgr);
                }
            } else {
                throw new IllegalStateException("FwdPassType not supported for this method: " + fwdPassType);
            }

            if(input == null){
                throw new IllegalStateException("Layer " + i + " returned null activations");
            }

            //Validation: Exception if invalid (bad layer implementation)
            validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, false, "Feed forward to layer (training)");
            validateArrayWorkspaces(workspaceMgr, layers[i].input(), ArrayType.INPUT, i, false, "Feed forward to layer (training)");

            out.add(input);

            if(traceLog){
                log.trace("Completed forward pass: {} - {}", i, layers[i].getClass().getSimpleName());
            }
        }
    }

    return out;
}
 
Example 4
Source File: MultiLayerNetwork.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
private Pair<Gradient,INDArray> calculateGradientsHelper(INDArray features, INDArray label, INDArray fMask,
                                                         INDArray labelMask){
    setInput(features);
    setLabels(label);
    setLayerMaskArrays(fMask, labelMask);

    LayerWorkspaceMgr mgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE){
        mgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        mgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.INPUT, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.ACTIVATIONS, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.BP_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_BP_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();

        if(layerWiseConfigurations.getCacheMode() != null){
            //For now: store cache mode activations in activations workspace
            mgr.setWorkspace(ArrayType.FF_CACHE, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG);
        }
    }
    mgr.setHelperWorkspacePointers(helperWorkspaces);

    //Calculate activations (which are stored in each layer, and used in backprop)
    try(MemoryWorkspace ws = mgr.notifyScopeEntered(ArrayType.ACTIVATIONS)) {
        //First: do a feed-forward through the network
        //Note that we don't actually need to do the full forward pass through the output layer right now; but we do
        // need the input to the output layer to be set (such that backprop can be done)
        List<INDArray> activations = ffToLayerActivationsInWs(layers.length - 2, FwdPassType.STANDARD, false, input, mask, fMask);
        if (!trainingListeners.isEmpty()) {
            //TODO: We possibly do want output layer activations in some cases here...
            for (TrainingListener tl : trainingListeners) {
                tl.onForwardPass(this, activations);
            }
        }
        INDArray inputToOutputLayer = activations.get(activations.size() - 1);
        if (layerWiseConfigurations.getInputPreProcess(layers.length - 1) != null) {
            inputToOutputLayer = layerWiseConfigurations.getInputPreProcess(layers.length - 1)
                    .preProcess(inputToOutputLayer, getInputMiniBatchSize(), mgr);
            //Validate activations location
        }
        getOutputLayer().setInput(inputToOutputLayer, mgr);

        Pair<Gradient,INDArray> p = calcBackpropGradients(null, true, false, true);
        if(p.getSecond() != null){
            p.setSecond( p.getSecond().detach());
        }
        return p;
    }
}
 
Example 5
Source File: MultiLayerNetwork.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
private double scoreHelper(DataSet data, boolean training){
    boolean hasMaskArray = data.hasMaskArrays();
    if (hasMaskArray)
        setLayerMaskArrays(data.getFeaturesMaskArray(), data.getLabelsMaskArray());

    if (!(getOutputLayer() instanceof IOutputLayer)) {
        throw new IllegalStateException("Cannot calculate score if final layer is not an instance of IOutputLayer. " +
                "Final layer is of type: " + getOutputLayer().getClass());
    }

    WorkspaceMode wsm = (training ? layerWiseConfigurations.getTrainingWorkspaceMode() : layerWiseConfigurations.getInferenceWorkspaceMode());
    LayerWorkspaceMgr mgr;
    if(wsm == WorkspaceMode.NONE){
        mgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        mgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                //TODO we can probably optimize this
                .noWorkspaceFor(ArrayType.ACTIVATIONS)
                .noWorkspaceFor(ArrayType.INPUT)
                .build();
    }
    mgr.setHelperWorkspacePointers(helperWorkspaces);

    INDArray inputToOutputLayer = outputOfLayerDetached(training, FwdPassType.STANDARD,layers.length-2, data.getFeatures(),
            data.getFeaturesMaskArray(), data.getLabelsMaskArray(), null);

    if (data.getFeatures().size(0) > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();
    IOutputLayer ol = (IOutputLayer) getOutputLayer();
    if (getLayerWiseConfigurations().getInputPreProcess(layers.length - 1) != null) {
        inputToOutputLayer = getLayerWiseConfigurations().getInputPreProcess(layers.length - 1)
                .preProcess(inputToOutputLayer, (int) data.getFeatures().size(0), mgr);
    }
    ol.setInput(inputToOutputLayer, mgr); //Feedforward doesn't include output layer for efficiency
    ol.setLabels(data.getLabels());
    double score;
    try(MemoryWorkspace ws = mgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)) {
        score = ol.computeScore(calcRegularizationScore(true), training, mgr);
    }

    if (hasMaskArray)
        clearLayerMaskArrays();
    clearLayersStates();

    return score;
}