Java Code Examples for org.deeplearning4j.nn.workspace.LayerWorkspaceMgr#setHelperWorkspacePointers()

/**
 * Perform layerwise unsupervised training on a single pre-trainable layer in the network (VAEs, Autoencoders, etc)<br>
 * If the specified layer index (0 to numLayers - 1) is not a pretrainable layer, this is a no-op.
 *
 * @param layerIdx Index of the layer to train (0 to numLayers-1)
 * @param features Training data array
 */
public void pretrainLayer(int layerIdx, INDArray features) {
    setInput(features);
    setLayerMaskArrays(null, null);

    if (flattenedGradients == null) {
        initGradientsView();
    }
    if (layerIdx >= layers.length) {
        throw new IllegalArgumentException(
                "Cannot pretrain layer: layerIdx (" + layerIdx + ") >= numLayers (" + layers.length + ")");
    }

    LayerWorkspaceMgr workspaceMgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE){
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .defaultWorkspace(WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    Layer layer = layers[layerIdx];
    if (!layer.isPretrainLayer())
        return;

    //Do forward pass to the layer to be pretrained
    INDArray outputOfPrevLayer;
    if(layerIdx == 0) {
        outputOfPrevLayer = input;
    } else {
        //Yes, this part of training - but we'll do forward psas as inference mode when doing layerwise training
        // to effectively freeze earlier layers and not apply dropout etc
        outputOfPrevLayer = outputOfLayerDetached(false, FwdPassType.STANDARD, layerIndex-1, features, null, null, null);
    }

    try(MemoryWorkspace ws = workspaceMgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)) {
        if (layerWiseConfigurations.getInputPreProcess(layerIdx) != null) {

            if (input.size(0) > Integer.MAX_VALUE)
                throw new ND4JArraySizeException();
            outputOfPrevLayer = layerWiseConfigurations.getInputPreProcess(layerIdx).preProcess(outputOfPrevLayer, (int) input.size(0),
                    LayerWorkspaceMgr.noWorkspaces(helperWorkspaces));
        }

        layer.fit(outputOfPrevLayer, workspaceMgr);
    }
}
 

/**
 * Feed-forward through the network - returning all array activations in a list, detached from any workspace.
 * Note that no workspace should be active externally when calling this method (an exception will be thrown
 * if a workspace is open externally)
 *
 * @param train             Training mode (true) or test/inference mode (false)
 * @param fwdPassType       Type of forward pass to perform (STANDARD or RNN_ACTIVATE_WITH_STORED_STATE only)
 * @param storeLastForTBPTT ONLY used if fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE
 * @param layerIndex        Index (inclusive) to stop forward pass at. For all layers, use numLayers-1
 * @param input             Input to the network
 * @param fMask             Feature mask array. May be null.
 * @param lMask             Label mask array. May be null.
 * @param clearInputs       Whether the layer inputs should be cleared
 * @return List of activations (including the input), detached from any workspace
 */
protected synchronized List<INDArray> ffToLayerActivationsDetached(boolean train, @NonNull FwdPassType fwdPassType,
                                                      boolean storeLastForTBPTT, int layerIndex, @NonNull INDArray input,
                                                      INDArray fMask, INDArray lMask, boolean clearInputs){
    setInput(input);
    setLayerMaskArrays(fMask, lMask);

    //Verify that no workspace is open externally
    WorkspaceUtils.assertNoWorkspacesOpen("Expected no workspace active in ffToLayerActivationsDetached");

    LayerWorkspaceMgr workspaceMgr;
    WorkspaceMode wsm = (train ? layerWiseConfigurations.getTrainingWorkspaceMode() : layerWiseConfigurations.getInferenceWorkspaceMode());
    if(wsm == WorkspaceMode.NONE){
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .noWorkspaceFor(ArrayType.ACTIVATIONS)
                .with(ArrayType.INPUT, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();

        if(input.isAttached()){
            //Don't leverage out of async DataSetIterator workspaces
            workspaceMgr.setNoLeverageOverride(input.data().getParentWorkspace().getId());
        }

        if(!clearInputs){
            workspaceMgr.setScopedOutFor(ArrayType.INPUT);
        }
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    List<INDArray> out = new ArrayList<>();
    out.add(workspaceMgr.leverageTo(ArrayType.INPUT, input));    //Should  be unnecessary (and no op), if layer is implemented correctly

    for( int i=0; i<=layerIndex; i++ ){
        try(MemoryWorkspace wsFFWorking = workspaceMgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)){
            if (getLayerWiseConfigurations().getInputPreProcess(i) != null) {
                input = getLayerWiseConfigurations().getInputPreProcess(i).preProcess(input, getInputMiniBatchSize(), workspaceMgr);
                //Validation: Exception if invalid (bad preprocessor implementation)
                validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, true, "Feed forward to layer (inference)");
            }

            if(fwdPassType == FwdPassType.STANDARD){
                input = layers[i].activate(input, train, workspaceMgr);
            } else if (fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE) {
                if (layers[i] instanceof RecurrentLayer) {
                    input = ((RecurrentLayer) layers[i]).rnnActivateUsingStoredState(input, train,
                            storeLastForTBPTT, workspaceMgr);
                } else if(layers[i] instanceof BaseWrapperLayer && ((BaseWrapperLayer)layers[i]).getUnderlying() instanceof RecurrentLayer) {
                    RecurrentLayer rl = (RecurrentLayer) ((BaseWrapperLayer)layers[i]).getUnderlying();
                    input = rl.rnnActivateUsingStoredState(input, train,storeLastForTBPTT, workspaceMgr);
                } else if (layers[i] instanceof MultiLayerNetwork) {
                    List<INDArray> temp = ((MultiLayerNetwork) layers[i]).rnnActivateUsingStoredState(input, train, storeLastForTBPTT);
                    input = temp.get(temp.size() - 1);
                } else {
                    input = layers[i].activate(input, train, workspaceMgr);
                }
            } else {
                throw new IllegalStateException("Forward pass type not supported for this method: " + fwdPassType);
            }

            //Validation: Exception if invalid (bad layer implementation)
            validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, false, "Feed forward to layer (inference)");

            out.add(input);
        }
        if(clearInputs) {
            layers[i].clear();
        }
    }

    return out;
}
 

/**
 * Feed-forward through the network at training time - returning a list of all activations in a workspace (WS_ALL_LAYERS_ACT)
 * if workspaces are enabled for training; or detached if no workspaces are used.<br>
 * Note: if using workspaces for training, this method requires that WS_ALL_LAYERS_ACT is open externally.<br>
 * If using NO workspaces, requires that no external workspace is open<br>
 * Note that this method does NOT clear the inputs to each layer - instead, they are in the WS_ALL_LAYERS_ACT workspace
 * for use in later backprop.
 *
 * @param layerIndex        Index (inclusive) to stop forward pass at. For all layers, use numLayers-1
 * @param fwdPassType       Type of forward pass to perform (STANDARD or RNN_ACTIVATE_WITH_STORED_STATE only)
 * @param storeLastForTBPTT ONLY used if fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE
 * @param input             Input to network
 * @param fMask             Feature mask array. May be null
 * @param lMask             Label mask aray. May be null.
 * @return
 */
protected synchronized List<INDArray> ffToLayerActivationsInWs(int layerIndex, @NonNull FwdPassType fwdPassType, boolean storeLastForTBPTT,
                                                  @NonNull INDArray input, INDArray fMask, INDArray lMask){
    setInput(input);
    setLayerMaskArrays(fMask, lMask);

    LayerWorkspaceMgr workspaceMgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE){
        WorkspaceUtils.assertNoWorkspacesOpen("Expected no workspace active in ffToLayerActivationsInWs when training workspace is set to NONE");
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.INPUT, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.ACTIVATIONS, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();

        if(input.isAttached()){
            //Don't leverage out of async DataSetIterator workspaces
            workspaceMgr.setNoLeverageOverride(input.data().getParentWorkspace().getId());
        }

        if(layerWiseConfigurations.getCacheMode() != CacheMode.NONE){
            //For now: store cache mode activations in activations workspace
            workspaceMgr.setWorkspace(ArrayType.FF_CACHE, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG);
            workspaceMgr.setWorkspace(ArrayType.BP_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG);
        }

        WorkspaceUtils.assertOpenAndActive(WS_ALL_LAYERS_ACT, "ffToLayerActivationsInWs method requires workspace WS_ALL_LAYERS_ACT to be open");
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    List<INDArray> out = new ArrayList<>();
    out.add(workspaceMgr.leverageTo(ArrayType.INPUT, input));    //Probably unnecessary usually

    boolean traceLog = log.isTraceEnabled();

    for( int i=0; i<=layerIndex; i++ ){
        try(MemoryWorkspace wsFFWorking = workspaceMgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)){
            if (getLayerWiseConfigurations().getInputPreProcess(i) != null) {
                input = getLayerWiseConfigurations().getInputPreProcess(i).preProcess(input, getInputMiniBatchSize(), workspaceMgr);
                //Validation: Exception if invalid (bad preprocessor implementation)
                validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, true, "Feed forward to layer (training)");
            }

            if(traceLog){
                log.trace("About to forward pass: {} - {}", i, layers[i].getClass().getSimpleName());
            }

            if(fwdPassType == FwdPassType.STANDARD){
                input = layers[i].activate(input, true, workspaceMgr);
            } else if(fwdPassType == FwdPassType.RNN_ACTIVATE_WITH_STORED_STATE){
                if (layers[i] instanceof RecurrentLayer) {
                    input = ((RecurrentLayer) layers[i]).rnnActivateUsingStoredState(input, true, storeLastForTBPTT, workspaceMgr);
                }else if(layers[i] instanceof BaseWrapperLayer && ((BaseWrapperLayer)layers[i]).getUnderlying() instanceof RecurrentLayer) {
                    RecurrentLayer rl = (RecurrentLayer) ((BaseWrapperLayer)layers[i]).getUnderlying();
                    input = rl.rnnActivateUsingStoredState(input, true, storeLastForTBPTT, workspaceMgr);
                } else if (layers[i] instanceof MultiLayerNetwork) {
                    List<INDArray> temp = ((MultiLayerNetwork) layers[i]).rnnActivateUsingStoredState(input, true, storeLastForTBPTT);
                    input = temp.get(temp.size() - 1);
                } else {
                    input = layers[i].activate(input, true, workspaceMgr);
                }
            } else {
                throw new IllegalStateException("FwdPassType not supported for this method: " + fwdPassType);
            }

            if(input == null){
                throw new IllegalStateException("Layer " + i + " returned null activations");
            }

            //Validation: Exception if invalid (bad layer implementation)
            validateArrayWorkspaces(workspaceMgr, input, ArrayType.ACTIVATIONS, i, false, "Feed forward to layer (training)");
            validateArrayWorkspaces(workspaceMgr, layers[i].input(), ArrayType.INPUT, i, false, "Feed forward to layer (training)");

            out.add(input);

            if(traceLog){
                log.trace("Completed forward pass: {} - {}", i, layers[i].getClass().getSimpleName());
            }
        }
    }

    return out;
}
 

private Pair<Gradient,INDArray> calculateGradientsHelper(INDArray features, INDArray label, INDArray fMask,
                                                         INDArray labelMask){
    setInput(features);
    setLabels(label);
    setLayerMaskArrays(fMask, labelMask);

    LayerWorkspaceMgr mgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE){
        mgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        mgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.INPUT, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.ACTIVATIONS, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.BP_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_BP_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                .build();

        if(layerWiseConfigurations.getCacheMode() != null){
            //For now: store cache mode activations in activations workspace
            mgr.setWorkspace(ArrayType.FF_CACHE, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG);
        }
    }
    mgr.setHelperWorkspacePointers(helperWorkspaces);

    //Calculate activations (which are stored in each layer, and used in backprop)
    try(MemoryWorkspace ws = mgr.notifyScopeEntered(ArrayType.ACTIVATIONS)) {
        //First: do a feed-forward through the network
        //Note that we don't actually need to do the full forward pass through the output layer right now; but we do
        // need the input to the output layer to be set (such that backprop can be done)
        List<INDArray> activations = ffToLayerActivationsInWs(layers.length - 2, FwdPassType.STANDARD, false, input, mask, fMask);
        if (!trainingListeners.isEmpty()) {
            //TODO: We possibly do want output layer activations in some cases here...
            for (TrainingListener tl : trainingListeners) {
                tl.onForwardPass(this, activations);
            }
        }
        INDArray inputToOutputLayer = activations.get(activations.size() - 1);
        if (layerWiseConfigurations.getInputPreProcess(layers.length - 1) != null) {
            inputToOutputLayer = layerWiseConfigurations.getInputPreProcess(layers.length - 1)
                    .preProcess(inputToOutputLayer, getInputMiniBatchSize(), mgr);
            //Validate activations location
        }
        getOutputLayer().setInput(inputToOutputLayer, mgr);

        Pair<Gradient,INDArray> p = calcBackpropGradients(null, true, false, true);
        if(p.getSecond() != null){
            p.setSecond( p.getSecond().detach());
        }
        return p;
    }
}
 

private void fitHelper(INDArray features, INDArray labels, INDArray featuresMask, INDArray labelsMask){
    if(numParams() == 0){
        //No op: can't fit a network with 0 parameters
        return;
    }

    setInput(features);
    setLabels(labels);
    this.setLayerMaskArrays(featuresMask, labelsMask);
    update(TaskUtils.buildTask(features, labels));

    LayerWorkspaceMgr workspaceMgr;
    if(layerWiseConfigurations.getTrainingWorkspaceMode() == null){
        workspaceMgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        workspaceMgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.INPUT, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .with(ArrayType.ACTIVATIONS, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                //Note for updater working memory, we have the option to re-use WS_ALL_LAYERS_ACT or FF/BP_WORKING_MEM
                // these should be closed by the time updaters are executed
                //Generally, WS_ALL_LAYERS_ACT will be the larger of the two, so we'll use this
                .with(ArrayType.UPDATER_WORKING_MEM, WS_ALL_LAYERS_ACT, WS_ALL_LAYERS_ACT_CONFIG)
                .build();
    }
    workspaceMgr.setHelperWorkspacePointers(helperWorkspaces);

    if (layerWiseConfigurations.getBackpropType() == BackpropType.TruncatedBPTT) {
        doTruncatedBPTT(features, labels, featuresMask, labelsMask, workspaceMgr);
    } else {
        if (solver == null) {
            try (MemoryWorkspace wsO = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
                solver = new Solver.Builder().configure(conf()).listeners(getListeners()).model(this).build();
            }
        }
        //TODO CACHE WORKSPACE, IF USED???
        solver.optimize(workspaceMgr);
    }

    clearLayerMaskArrays();
    clearLayersStates();
    synchronizeIterEpochCounts();
}
 

private double scoreHelper(DataSet data, boolean training){
    boolean hasMaskArray = data.hasMaskArrays();
    if (hasMaskArray)
        setLayerMaskArrays(data.getFeaturesMaskArray(), data.getLabelsMaskArray());

    if (!(getOutputLayer() instanceof IOutputLayer)) {
        throw new IllegalStateException("Cannot calculate score if final layer is not an instance of IOutputLayer. " +
                "Final layer is of type: " + getOutputLayer().getClass());
    }

    WorkspaceMode wsm = (training ? layerWiseConfigurations.getTrainingWorkspaceMode() : layerWiseConfigurations.getInferenceWorkspaceMode());
    LayerWorkspaceMgr mgr;
    if(wsm == WorkspaceMode.NONE){
        mgr = LayerWorkspaceMgr.noWorkspaces();
    } else {
        mgr = LayerWorkspaceMgr.builder()
                .with(ArrayType.FF_WORKING_MEM, WS_LAYER_WORKING_MEM, WS_LAYER_WORKING_MEM_CONFIG)
                .with(ArrayType.RNN_FF_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM, WS_RNN_LOOP_WORKING_MEM_CONFIG)
                //TODO we can probably optimize this
                .noWorkspaceFor(ArrayType.ACTIVATIONS)
                .noWorkspaceFor(ArrayType.INPUT)
                .build();
    }
    mgr.setHelperWorkspacePointers(helperWorkspaces);

    INDArray inputToOutputLayer = outputOfLayerDetached(training, FwdPassType.STANDARD,layers.length-2, data.getFeatures(),
            data.getFeaturesMaskArray(), data.getLabelsMaskArray(), null);

    if (data.getFeatures().size(0) > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();
    IOutputLayer ol = (IOutputLayer) getOutputLayer();
    if (getLayerWiseConfigurations().getInputPreProcess(layers.length - 1) != null) {
        inputToOutputLayer = getLayerWiseConfigurations().getInputPreProcess(layers.length - 1)
                .preProcess(inputToOutputLayer, (int) data.getFeatures().size(0), mgr);
    }
    ol.setInput(inputToOutputLayer, mgr); //Feedforward doesn't include output layer for efficiency
    ol.setLabels(data.getLabels());
    double score;
    try(MemoryWorkspace ws = mgr.notifyScopeEntered(ArrayType.FF_WORKING_MEM)) {
        score = ol.computeScore(calcRegularizationScore(true), training, mgr);
    }

    if (hasMaskArray)
        clearLayerMaskArrays();
    clearLayersStates();

    return score;
}