Java Code Examples for org.deeplearning4j.nn.conf.NeuralNetConfiguration#getLayer()

@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramsView, boolean initializeParams) {
    Map<String, INDArray> params = super.init(conf, paramsView, initializeParams);

    org.deeplearning4j.nn.conf.layers.BasePretrainNetwork layerConf =
                    (org.deeplearning4j.nn.conf.layers.BasePretrainNetwork) conf.getLayer();
    val nIn = layerConf.getNIn();
    val nOut = layerConf.getNOut();
    val nWeightParams = nIn * nOut;

    INDArray visibleBiasView = paramsView.get(NDArrayIndex.interval(0,0,true),
                    NDArrayIndex.interval(nWeightParams + nOut, nWeightParams + nOut + nIn));
    params.put(VISIBLE_BIAS_KEY, createVisibleBias(conf, visibleBiasView, initializeParams));
    conf.addVariable(VISIBLE_BIAS_KEY);

    return params;
}
 

@Test
public void testForwardPass() {

    int[][] encLayerSizes = new int[][] {{12}, {12, 13}, {12, 13, 14}};
    for (int i = 0; i < encLayerSizes.length; i++) {

        MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder().list().layer(0,
                        new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().nIn(10)
                                        .nOut(5).encoderLayerSizes(encLayerSizes[i]).decoderLayerSizes(13).build())
                        .build();

        NeuralNetConfiguration c = mlc.getConf(0);
        org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder vae =
                        (org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder) c.getLayer();

        MultiLayerNetwork net = new MultiLayerNetwork(mlc);
        net.init();

        INDArray in = Nd4j.rand(1, 10);

        //        net.output(in);
        List<INDArray> out = net.feedForward(in);
        assertArrayEquals(new long[] {1, 10}, out.get(0).shape());
        assertArrayEquals(new long[] {1, 5}, out.get(1).shape());
    }
}
 

@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {

    DepthwiseConvolution2D layerConf = (DepthwiseConvolution2D) conf.getLayer();

    int[] kernel = layerConf.getKernelSize();
    val nIn = layerConf.getNIn();
    val depthMultiplier = layerConf.getDepthMultiplier();
    val nOut = layerConf.getNOut();

    Map<String, INDArray> out = new LinkedHashMap<>();

    val depthWiseParams = numDepthWiseParams(layerConf);
    val biasParams = numBiasParams(layerConf);

    INDArray depthWiseWeightGradientView = gradientView.get(
            NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(biasParams, biasParams + depthWiseParams))
            .reshape('c', kernel[0], kernel[1], nIn, depthMultiplier);
    out.put(WEIGHT_KEY, depthWiseWeightGradientView);

    if(layerConf.hasBias()){
        INDArray biasGradientView = gradientView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nOut));
        out.put(BIAS_KEY, biasGradientView);
    }
    return out;
}
 

public BidirectionalLayer(@NonNull NeuralNetConfiguration conf, @NonNull Layer fwd, @NonNull Layer bwd, @NonNull INDArray paramsView) {
    this.conf = conf;
    this.fwd = fwd;
    this.bwd = bwd;
    this.layerConf = (Bidirectional) conf.getLayer();
    this.paramsView = paramsView;
}
 

@Override
public long numParams(NeuralNetConfiguration conf) {
    FeedForwardLayer layerConf = (FeedForwardLayer) conf.getLayer();
    return super.numParams(conf) + numberOfUsers * layerConf.getNOut(); // add
    // another
    // user
    // weight
    // matrix
}
 

@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramsView, boolean initializeParams) {
    Layer orig = conf.getLayer();
    Layer l = underlying(conf.getLayer());
    conf.setLayer(l);
    Map<String,INDArray> m = l.initializer().init(conf, paramsView, initializeParams);
    conf.setLayer(orig);
    return m;
}
 

@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    FrozenLayerWithBackprop fl = (FrozenLayerWithBackprop) conf.getLayer();
    Layer innerLayer = fl.getUnderlying();
    ParamInitializer initializer = innerLayer.initializer();
    conf.setLayer(innerLayer);
    Map<String, INDArray> m = initializer.getGradientsFromFlattened(conf, gradientView);
    conf.setLayer(fl);
    return m;
}
 

@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramsView, boolean initializeParams) {
    FrozenLayerWithBackprop fl = (FrozenLayerWithBackprop) conf.getLayer();
    Layer innerLayer = fl.getUnderlying();
    ParamInitializer initializer = innerLayer.initializer();
    conf.setLayer(innerLayer);
    Map<String, INDArray> m = initializer.init(conf, paramsView, initializeParams);
    conf.setLayer(fl);

    return m;
}
 

@Test
public void testPretrainSimple() {

    int inputSize = 3;

    MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .nIn(inputSize).nOut(4).encoderLayerSizes(5).decoderLayerSizes(6).build())
                    .build();

    NeuralNetConfiguration c = mlc.getConf(0);
    org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder vae =
                    (org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder) c.getLayer();

    long allParams = vae.initializer().numParams(c);

    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();
    net.initGradientsView(); //TODO this should happen automatically

    Map<String, INDArray> paramTable = net.getLayer(0).paramTable();
    Map<String, INDArray> gradTable =
                    ((org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) net.getLayer(0))
                                    .getGradientViews();

    assertEquals(paramTable.keySet(), gradTable.keySet());
    for (String s : paramTable.keySet()) {
        assertEquals(paramTable.get(s).length(), gradTable.get(s).length());
        assertArrayEquals(paramTable.get(s).shape(), gradTable.get(s).shape());
    }

    System.out.println("Num params: " + net.numParams());

    INDArray data = Nd4j.rand(1, inputSize);


    net.pretrainLayer(0, data);
}
 

@Override
public long numParams(NeuralNetConfiguration conf) {
    org.deeplearning4j.nn.conf.layers.FeedForwardLayer layerConf =
                    (org.deeplearning4j.nn.conf.layers.FeedForwardLayer) conf.getLayer();
    val nIn = layerConf.getNIn();
    val nOut = layerConf.getNOut(); // also equal to numClasses
    return nIn * nOut + nOut + nIn * nOut; //weights + bias + embeddings
}
 

@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    org.deeplearning4j.nn.conf.layers.GravesLSTM layerConf =
                    (org.deeplearning4j.nn.conf.layers.GravesLSTM) conf.getLayer();

    val nL = layerConf.getNOut(); //i.e., n neurons in this layer
    val nLast = layerConf.getNIn(); //i.e., n neurons in previous layer

    val length = numParams(conf);
    if (gradientView.length() != length)
        throw new IllegalStateException(
                        "Expected gradient view of length " + length + ", got length " + gradientView.length());

    val nParamsIn = nLast * (4 * nL);
    val nParamsRecurrent = nL * (4 * nL + 3);
    val nBias = 4 * nL;
    INDArray inputWeightGradView = gradientView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nParamsIn))
                    .reshape('f', nLast, 4 * nL);
    INDArray recurrentWeightGradView = gradientView
                    .get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(nParamsIn, nParamsIn + nParamsRecurrent))
                    .reshape('f', nL, 4 * nL + 3);
    INDArray biasGradView = gradientView.get(NDArrayIndex.interval(0,0,true),
                    NDArrayIndex.interval(nParamsIn + nParamsRecurrent, nParamsIn + nParamsRecurrent + nBias)); //already a row vector

    Map<String, INDArray> out = new LinkedHashMap<>();
    out.put(INPUT_WEIGHT_KEY, inputWeightGradView);
    out.put(RECURRENT_WEIGHT_KEY, recurrentWeightGradView);
    out.put(BIAS_KEY, biasGradView);

    return out;
}
 

@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    FrozenLayer fl = (FrozenLayer) conf.getLayer();
    Layer innerLayer = fl.getLayer();
    ParamInitializer initializer = innerLayer.initializer();
    conf.setLayer(innerLayer);
    Map<String, INDArray> m = initializer.getGradientsFromFlattened(conf, gradientView);
    conf.setLayer(fl);
    return m;
}
 

public GlobalPoolingLayer(NeuralNetConfiguration conf, DataType dataType) {
    super(conf, dataType);

    org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer layerConf =
            (org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer) conf.getLayer();

    poolingDimensions = layerConf.getPoolingDimensions();
    poolingType = layerConf.getPoolingType();
    pNorm = layerConf.getPnorm();
}
 

protected INDArray createWeightMatrix(NeuralNetConfiguration configuration, INDArray view, boolean initialize) {
    FeedForwardLayer layerConfiguration = (FeedForwardLayer) configuration.getLayer();
    if (initialize) {
        Distribution distribution = Distributions.createDistribution(layerConfiguration.getDist());
        return super.createWeightMatrix(numberOfFeatures, layerConfiguration.getNOut(), layerConfiguration.getWeightInit(), distribution, view, true);
    } else {
        return super.createWeightMatrix(numberOfFeatures, layerConfiguration.getNOut(), null, null, view, false);
    }
}
 

public static GraphInfo buildGraphInfo(MultiLayerConfiguration config) {
    List<String> vertexNames = new ArrayList<>();
    List<String> originalVertexName = new ArrayList<>();
    List<String> layerTypes = new ArrayList<>();
    List<List<Integer>> layerInputs = new ArrayList<>();
    List<Map<String, String>> layerInfo = new ArrayList<>();
    vertexNames.add("Input");
    originalVertexName.add(null);
    layerTypes.add("Input");
    layerInputs.add(Collections.emptyList());
    layerInfo.add(Collections.emptyMap());


    List<NeuralNetConfiguration> list = config.getConfs();
    int layerIdx = 1;
    for (NeuralNetConfiguration c : list) {
        Layer layer = c.getLayer();
        String layerName = layer.getLayerName();
        if (layerName == null)
            layerName = "layer" + layerIdx;
        vertexNames.add(layerName);
        originalVertexName.add(String.valueOf(layerIdx - 1));

        String layerType = c.getLayer().getClass().getSimpleName().replaceAll("Layer$", "");
        layerTypes.add(layerType);

        layerInputs.add(Collections.singletonList(layerIdx - 1));
        layerIdx++;

        //Extract layer info
        Map<String, String> map = getLayerInfo(c, layer);
        layerInfo.add(map);
    }

    return new GraphInfo(vertexNames, layerTypes, layerInputs, layerInfo, originalVertexName);
}
 

@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramsView, boolean initializeParams) {
    Map<String, INDArray> params = Collections.synchronizedMap(new LinkedHashMap<String, INDArray>());

    org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM layerConf =
            (org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM) conf.getLayer();
    double forgetGateInit = layerConf.getForgetGateBiasInit();

    val nL = layerConf.getNOut(); //i.e., n neurons in this layer
    val nLast = layerConf.getNIn(); //i.e., n neurons in previous layer

    conf.addVariable(INPUT_WEIGHT_KEY_FORWARDS);
    conf.addVariable(RECURRENT_WEIGHT_KEY_FORWARDS);
    conf.addVariable(BIAS_KEY_FORWARDS);
    conf.addVariable(INPUT_WEIGHT_KEY_BACKWARDS);
    conf.addVariable(RECURRENT_WEIGHT_KEY_BACKWARDS);
    conf.addVariable(BIAS_KEY_BACKWARDS);

    val nParamsInput = nLast * (4 * nL);
    val nParamsRecurrent = nL * (4 * nL + 3);
    val nBias = 4 * nL;

    val rwFOffset = nParamsInput;
    val bFOffset = rwFOffset + nParamsRecurrent;
    val iwROffset = bFOffset + nBias;
    val rwROffset = iwROffset + nParamsInput;
    val bROffset = rwROffset + nParamsRecurrent;

    INDArray iwF = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, rwFOffset));
    INDArray rwF = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(rwFOffset, bFOffset));
    INDArray bF = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(bFOffset, iwROffset));
    INDArray iwR = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(iwROffset, rwROffset));
    INDArray rwR = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(rwROffset, bROffset));
    INDArray bR = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(bROffset, bROffset + nBias));

    if (initializeParams) {
        bF.put(new INDArrayIndex[]{NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(nL, 2 * nL)},
                Nd4j.ones(1, nL).muli(forgetGateInit)); //Order: input, forget, output, input modulation, i.e., IFOG
        bR.put(new INDArrayIndex[]{NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(nL, 2 * nL)},
                Nd4j.ones(1, nL).muli(forgetGateInit));
    }
    /*The above line initializes the forget gate biases to specified value.
     * See Sutskever PhD thesis, pg19:
     * "it is important for [the forget gate activations] to be approximately 1 at the early stages of learning,
     *  which is accomplished by initializing [the forget gate biases] to a large value (such as 5). If it is
     *  not done, it will be harder to learn long range dependencies because the smaller values of the forget
     *  gates will create a vanishing gradients problem."
     *  http://www.cs.utoronto.ca/~ilya/pubs/ilya_sutskever_phd_thesis.pdf
     */

    if (initializeParams) {
        //As per standard LSTM
        val fanIn = nL;
        val fanOut = nLast + nL;
        val inputWShape = new long[]{nLast, 4 * nL};
        val recurrentWShape = new long[]{nL, 4 * nL + 3};

        params.put(INPUT_WEIGHT_KEY_FORWARDS, layerConf.getWeightInitFn().init(fanIn, fanOut, inputWShape,
                IWeightInit.DEFAULT_WEIGHT_INIT_ORDER, iwF));
        params.put(RECURRENT_WEIGHT_KEY_FORWARDS, layerConf.getWeightInitFn().init(fanIn, fanOut, recurrentWShape,
                IWeightInit.DEFAULT_WEIGHT_INIT_ORDER, rwF));
        params.put(BIAS_KEY_FORWARDS, bF);
        params.put(INPUT_WEIGHT_KEY_BACKWARDS, layerConf.getWeightInitFn().init(fanIn, fanOut, inputWShape,
                IWeightInit.DEFAULT_WEIGHT_INIT_ORDER, iwR));
        params.put(RECURRENT_WEIGHT_KEY_BACKWARDS, layerConf.getWeightInitFn().init(fanIn, fanOut, recurrentWShape,
                IWeightInit.DEFAULT_WEIGHT_INIT_ORDER, rwR));
        params.put(BIAS_KEY_BACKWARDS, bR);
    } else {
        params.put(INPUT_WEIGHT_KEY_FORWARDS, WeightInitUtil.reshapeWeights(new long[]{nLast, 4 * nL}, iwF));
        params.put(RECURRENT_WEIGHT_KEY_FORWARDS, WeightInitUtil.reshapeWeights(new long[]{nL, 4 * nL + 3}, rwF));
        params.put(BIAS_KEY_FORWARDS, bF);
        params.put(INPUT_WEIGHT_KEY_BACKWARDS, WeightInitUtil.reshapeWeights(new long[]{nLast, 4 * nL}, iwR));
        params.put(RECURRENT_WEIGHT_KEY_BACKWARDS, WeightInitUtil.reshapeWeights(new long[]{nL, 4 * nL + 3}, rwR));
        params.put(BIAS_KEY_BACKWARDS, bR);
    }

    return params;
}
 

@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    AbstractSameDiffLayer sd = (AbstractSameDiffLayer) conf.getLayer();
    return subsetAndReshape(sd.getLayerParams().getParameterKeys(), sd.getLayerParams().getParamShapes(),
            gradientView, sd);
}
 

@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramView, boolean initializeParams) {
    Map<String, INDArray> params = Collections.synchronizedMap(new LinkedHashMap<String, INDArray>());
    // TODO setup for RNN
    BatchNormalization layer = (BatchNormalization) conf.getLayer();
    val nOut = layer.getNOut();

    long meanOffset = 0;
    if (!layer.isLockGammaBeta()) { //No gamma/beta parameters when gamma/beta are locked
        INDArray gammaView = paramView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nOut));
        INDArray betaView = paramView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(nOut, 2 * nOut));

        params.put(GAMMA, createGamma(conf, gammaView, initializeParams));
        conf.addVariable(GAMMA);
        params.put(BETA, createBeta(conf, betaView, initializeParams));
        conf.addVariable(BETA);

        meanOffset = 2 * nOut;
    }

    INDArray globalMeanView =
                    paramView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(meanOffset, meanOffset + nOut));
    INDArray globalVarView = paramView.get(NDArrayIndex.interval(0,0,true),
                    NDArrayIndex.interval(meanOffset + nOut, meanOffset + 2 * nOut));

    if (initializeParams) {
        globalMeanView.assign(0);
        if(layer.isUseLogStd()){
            //Global log stdev: assign 0.0 as initial value (s=sqrt(v), and log10(s) = log10(sqrt(v)) -> log10(1) = 0
            globalVarView.assign(0);
        } else {
            //Global variance view: assign 1.0 as initial value
            globalVarView.assign(1);
        }
    }

    params.put(GLOBAL_MEAN, globalMeanView);
    conf.addVariable(GLOBAL_MEAN);
    if(layer.isUseLogStd()){
        params.put(GLOBAL_LOG_STD, globalVarView);
        conf.addVariable(GLOBAL_LOG_STD);
    } else {
        params.put(GLOBAL_VAR, globalVarView);
        conf.addVariable(GLOBAL_VAR);
    }

    return params;
}
 

protected INDArray createBias(NeuralNetConfiguration conf, INDArray biasParamView, boolean initializeParameters) {
    org.deeplearning4j.nn.conf.layers.FeedForwardLayer layerConf =
                    (org.deeplearning4j.nn.conf.layers.FeedForwardLayer) conf.getLayer();
    return createBias(layerConf.getNOut(), layerConf.getBiasInit(), biasParamView, initializeParameters);
}
 

@Override
public long numParams(NeuralNetConfiguration conf) {
    org.deeplearning4j.nn.conf.layers.BasePretrainNetwork layerConf =
                    (org.deeplearning4j.nn.conf.layers.BasePretrainNetwork) conf.getLayer();
    return super.numParams(conf) + layerConf.getNIn();
}