org.deeplearning4j.nn.params.DefaultParamInitializer Java Examples

@Override
public void initializeParameters(Map<String, INDArray> params) {
    String b = DefaultParamInitializer.BIAS_KEY;
    if(paramWeightInit != null && paramWeightInit.containsKey(b)){
        paramWeightInit.get(b).init(nIn, nOut, params.get(b).shape(), 'c', params.get(b));
    } else {
        params.get(DefaultParamInitializer.BIAS_KEY).assign(0);
    }

    String w = DefaultParamInitializer.WEIGHT_KEY;
    if(paramWeightInit != null && paramWeightInit.containsKey(w)){
        paramWeightInit.get(w).init(nIn, nOut, params.get(w).shape(), 'c', params.get(w));
    } else {
        initWeights(nIn, nOut, weightInit, params.get(DefaultParamInitializer.WEIGHT_KEY));
    }
}
 

@Test
public void testRNG() {
    DenseLayer layer = new DenseLayer.Builder().nIn(trainingSet.numInputs()).nOut(trainingSet.numOutcomes())
                    .weightInit(WeightInit.UNIFORM).activation(Activation.TANH).build();

    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(123)
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).layer(layer).build();

    long numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    Layer model = conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType());
    INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY);


    DenseLayer layer2 = new DenseLayer.Builder().nIn(trainingSet.numInputs()).nOut(trainingSet.numOutcomes())
                    .weightInit(WeightInit.UNIFORM).activation(Activation.TANH).build();
    NeuralNetConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(123)
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).layer(layer2).build();

    long numParams2 = conf2.getLayer().initializer().numParams(conf);
    INDArray params2 = Nd4j.create(1, numParams);
    Layer model2 = conf2.getLayer().instantiate(conf2, null, 0, params2, true, params.dataType());
    INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY);

    assertEquals(modelWeights, modelWeights2);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);
    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, delta.dataType(), new long[]{w.size(0), delta.size(0)}, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();

    //Normally we would clear weightNoiseParams here - but we want to reuse them for forward + backward + score
    // So this is instead done in MultiLayerNetwork/CompGraph backprop methods

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

/** Returns tuple: {Gradient,Delta,Output} given preOut */
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    //INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFunction(), maskArray);
    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.gemm(input.castTo(weightGradView.dataType()), delta, weightGradView, true, false, 1.0, 0.0); //Equivalent to:  weightGradView.assign(input.transpose().mmul(delta));         //TODO can we avoid cast?
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradView);

    if(hasBias()){
        INDArray biasGradView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGradView, 0); //biasGradView is initialized/zeroed first in sum op
        gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradView);
    }

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

@Test
public void testSetParams() {
    MultiLayerConfiguration conf =
                    new NeuralNetConfiguration.Builder()
                                    .list().layer(0,
                                                    new DenseLayer.Builder().nIn(4).nOut(3)
                                                            .activation(Activation.TANH).build())
                                    .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build())
                                    .build();

    MultiLayerNetwork network3 = new MultiLayerNetwork(conf);
    network3.init();

    INDArray params = network3.params();
    INDArray weights = network3.getLayer(0).getParam(DefaultParamInitializer.WEIGHT_KEY).dup();
    INDArray bias = network3.getLayer(0).getParam(DefaultParamInitializer.BIAS_KEY).dup();
    network3.setParameters(params);
    assertEquals(weights, network3.getLayer(0).getParam(DefaultParamInitializer.WEIGHT_KEY));
    assertEquals(bias, network3.getLayer(0).getParam(DefaultParamInitializer.BIAS_KEY));
    INDArray params4 = network3.params();
    assertEquals(params, params4);
}
 

/**
 * Set weights for layer.
 *
 * @param weights Dense layer weights
 */
@Override
public void setWeights(Map<String, INDArray> weights) throws InvalidKerasConfigurationException {
    this.weights = new HashMap<>();
    if (weights.containsKey(conf.getKERAS_PARAM_NAME_W()))
        this.weights.put(DefaultParamInitializer.WEIGHT_KEY, weights.get(conf.getKERAS_PARAM_NAME_W()));
    else
        throw new InvalidKerasConfigurationException(
                "Parameter " + conf.getKERAS_PARAM_NAME_W() + " does not exist in weights");
    if (hasBias) {
        if (weights.containsKey(conf.getKERAS_PARAM_NAME_B()))
            this.weights.put(DefaultParamInitializer.BIAS_KEY, weights.get(conf.getKERAS_PARAM_NAME_B()));
        else
            throw new InvalidKerasConfigurationException(
                    "Parameter " + conf.getKERAS_PARAM_NAME_B() + " does not exist in weights");
    }
    removeDefaultWeights(weights, conf);
}
 

@Test
public void testSameDiffDenseBasic() {
    int nIn = 3;
    int nOut = 4;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Map<String, INDArray> pt1 = net.getLayer(0).paramTable();
    assertNotNull(pt1);
    assertEquals(2, pt1.size());
    assertNotNull(pt1.get(DefaultParamInitializer.WEIGHT_KEY));
    assertNotNull(pt1.get(DefaultParamInitializer.BIAS_KEY));

    assertArrayEquals(new long[]{nIn, nOut}, pt1.get(DefaultParamInitializer.WEIGHT_KEY).shape());
    assertArrayEquals(new long[]{1, nOut}, pt1.get(DefaultParamInitializer.BIAS_KEY).shape());
}
 

@Test
public void testEmbeddingLayerConfig() {

    for (boolean hasBias : new boolean[]{true, false}) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(0, new EmbeddingLayer.Builder().hasBias(hasBias).nIn(10).nOut(5).build())
                .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l0 = net.getLayer(0);

        assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingLayer.class, l0.getClass());
        assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn());
        assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut());

        INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY);
        INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY);
        assertArrayEquals(new long[]{10, 5}, weights.shape());
        if (hasBias) {
            assertArrayEquals(new long[]{1, 5}, bias.shape());
        }
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray previous, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); // Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);
    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, new long[] { w.size(0), delta.size(0) }, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();

    // Normally we would clear weightNoiseParams here - but we want to reuse them
    // for forward + backward + score
    // So this is instead done in MultiLayerNetwork/CompGraph backprop methods

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    // INDArray delta = lossFunction.computeGradient(labels2d, preOut,
    // layerConf().getActivationFunction(), maskArray);
    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.gemm(input, delta, weightGradView, true, false, 1.0, 0.0); // Equivalent to: weightGradView.assign(input.transpose().mmul(delta));
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradView);

    if (hasBias()) {
        INDArray biasGradView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGradView, 0); // biasGradView is initialized/zeroed first in sum op
        gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradView);
    }

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

@Override
public void initializeParameters(Map<String,INDArray> params){
    for(Map.Entry<String,INDArray> e : params.entrySet()){
        if(paramWeightInit != null && paramWeightInit.containsKey(e.getKey())){
            paramWeightInit.get(e.getKey()).init(nIn, nOut, e.getValue().shape(), 'c', e.getValue());
        } else {
            if(DefaultParamInitializer.BIAS_KEY.equals(e.getKey())){
                e.getValue().assign(0.0);
            } else {
                //Normally use 'c' order, but use 'f' for direct comparison to DL4J DenseLayer
                WeightInitUtil.initWeights(nIn, nOut, new long[]{nIn, nOut}, weightInit, null, 'f', e.getValue());
            }
        }
    }
}
 

@Override
public SDVariable defineVertex(SameDiff sameDiff, Map<String, SDVariable> layerInput, Map<String, SDVariable> paramTable, Map<String, SDVariable> maskVars) {
    SDVariable weights = paramTable.get(DefaultParamInitializer.WEIGHT_KEY);
    SDVariable bias = paramTable.get(DefaultParamInitializer.BIAS_KEY);

    SDVariable mmul = sameDiff.mmul("mmul", layerInput.get("in"), weights);
    SDVariable z = mmul.add("z", bias);
    return activation.asSameDiff("out", sameDiff, z);
}
 

@Override
public SDVariable defineLayer(SameDiff sd, SDVariable layerInput, Map<String, SDVariable> paramTable, SDVariable mask) {
    SDVariable weights = paramTable.get(DefaultParamInitializer.WEIGHT_KEY);
    SDVariable bias = paramTable.get(DefaultParamInitializer.BIAS_KEY);

    SDVariable mmul = sd.mmul("mmul", layerInput, weights);
    SDVariable z = mmul.add("z", bias);
    return activation.asSameDiff("out", sd, z);
}
 

@Before
public void beforeDo() {
    gradients = Nd4j.ones(1, nIn * nOut + nOut);
    weightGradient = gradients.get(point(0), interval(0, nIn * nOut));
    biasGradient = gradients.get(point(0), interval(nIn * nOut, nIn * nOut + nOut));
    gradient.setGradientFor(DefaultParamInitializer.WEIGHT_KEY, weightGradient);
    gradient.setGradientFor(DefaultParamInitializer.BIAS_KEY, biasGradient);
    gradient.setFlattenedGradient(gradients);
}
 

@Test
public void testEmbeddingSequenceLayerConfig() {

    int inputLength = 6;
    int nIn = 10;
    int embeddingDim = 5;
    int nout = 4;

    for (boolean hasBias : new boolean[]{true, false}) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(new EmbeddingSequenceLayer.Builder().hasBias(hasBias)
                        .inputLength(inputLength).nIn(nIn).nOut(embeddingDim).build())
                .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nout).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l0 = net.getLayer(0);

        assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingSequenceLayer.class, l0.getClass());
        assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn());
        assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut());

        INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY);
        INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY);
        assertArrayEquals(new long[]{10, 5}, weights.shape());
        if (hasBias) {
            assertArrayEquals(new long[]{1, 5}, bias.shape());
        }
    }
}
 

@Override
public ParamInitializer initializer() {
    //This method returns the parameter initializer for this type of layer
    //In this case, we can use the DefaultParamInitializer, which is the same one used for DenseLayer
    //For more complex layers, you may need to implement a custom parameter initializer
    //See the various parameter initializers here:
    //https://github.com/deeplearning4j/deeplearning4j/tree/master/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/params

    return DefaultParamInitializer.getInstance();
}
 

@Override
public SDVariable defineLayer(SameDiff sd, SDVariable layerInput, Map<String, SDVariable> paramTable, SDVariable mask) {
    SDVariable weights = paramTable.get(DefaultParamInitializer.WEIGHT_KEY);
    SDVariable bias = paramTable.get(DefaultParamInitializer.BIAS_KEY);

    SDVariable mmul = sd.mmul("mmul", layerInput, weights);
    SDVariable z = mmul.add("z", bias);
    return activation.asSameDiff("out", sd, z);
}
 

private Gradient createPrevGradient() {
    Gradient gradient = new DefaultGradient();
    INDArray pseudoGradients = Nd4j.ones(nExamples, nChannelsIn, inputHeight, inputWidth);

    gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, pseudoGradients);
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, pseudoGradients);
    return gradient;
}
 

@Override
public INDArray preOutput(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    applyDropOutIfNecessary(training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);

    INDArray ret = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.size(0), W.size(1));
    ret.assign(0F);
    for (int row = 0; row < input.rows(); row++) {
        for (int column = 0; column < W.columns(); column++) {
            float value = 0F;
            int cursor = 0;
            for (int index = 0; index < input.columns(); index++) {
                value += W.getFloat(cursor + input.getInt(row, index), column);
                cursor += dimensionSizes[index];
            }
            ret.put(row, column, value);
        }
    }

    if (hasBias()) {
        ret.addiRowVector(b);
    }

    if (maskArray != null) {
        applyMask(ret);
    }

    return ret;
}
 

@Test
public void testSetSeedNormalized() {
    Nd4j.getRandom().setSeed(123);

    Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitXavier(), true);
    INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.getRandom().setSeed(123);

    Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitXavier(), true);
    INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY);
    assertEquals(modelWeights, modelWeights2);
}
 

@Override
public INDArray preOutput(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    applyDropOutIfNecessary(training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);
    INDArray U = getParamWithNoise(CDAEParameter.USER_KEY, training, workspaceMgr);
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);

    // Input validation:
    if (input.rank() != 2 || input.columns() != W.rows()) {
        if (input.rank() != 2) {
            throw new DL4JInvalidInputException("Input that is not a matrix; expected matrix (rank 2), got rank " + input.rank() + " array with shape " + Arrays.toString(input.shape()) + ". Missing preprocessor or wrong input type? " + layerId());
        }
        throw new DL4JInvalidInputException("Input size (" + input.columns() + " columns; shape = " + Arrays.toString(input.shape()) + ") is invalid: does not match layer input size (layer # inputs = " + W.size(0) + ") " + layerId());
    }

    INDArray ret = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.size(0), W.size(1));
    input.mmuli(W, ret);
    ret.addi(U);
    if (hasBias()) {
        ret.addiRowVector(b);
    }

    if (maskArray != null) {
        applyMask(ret);
    }

    return ret;
}
 

@Test
public void testSetSeedDistribution() {
    Nd4j.getRandom().setSeed(123);

    Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(),
            new WeightInitDistribution(new NormalDistribution(1, 1)), true);
    INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.getRandom().setSeed(123);

    Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(),
            new WeightInitDistribution(new NormalDistribution(1, 1)), true);
    INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY);

    assertEquals(modelWeights, modelWeights2);
}
 

@Test
public void testSetSeedXavier() {
    Nd4j.getRandom().setSeed(123);

    Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitUniform(), true);
    INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.getRandom().setSeed(123);

    Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitUniform(), true);
    INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY);

    assertEquals(modelWeights, modelWeights2);
}
 

private static Map<String, ParameterConfigurator> getConfigurators(MathCache factory, AbstractLayer<?> layer) {
    Map<String, ParameterConfigurator> configurators = new HashMap<>();
    CopyParameterFactory weight = new CopyParameterFactory(getMatrix(factory, layer.getParam(DefaultParamInitializer.WEIGHT_KEY)));
    configurators.put(WeightLayer.WEIGHT_KEY, new ParameterConfigurator(l1Regularization, l2Regularization, weight));
    CopyParameterFactory bias = new CopyParameterFactory(getMatrix(factory, layer.getParam(DefaultParamInitializer.BIAS_KEY)));
    configurators.put(WeightLayer.BIAS_KEY, new ParameterConfigurator(l1Regularization, l2Regularization, bias));
    return configurators;
}
 

@Override
protected List<KeyValue<String, String>> getGradients() {
    List<KeyValue<String, String>> gradients = new LinkedList<>();
    gradients.add(new KeyValue<>(DefaultParamInitializer.WEIGHT_KEY, WeightLayer.WEIGHT_KEY));
    gradients.add(new KeyValue<>(DefaultParamInitializer.BIAS_KEY, WeightLayer.BIAS_KEY));
    return gradients;
}
 

@Override
protected Layer getNewFunction(AbstractLayer<?> layer) {
    Map<String, ParameterConfigurator> configurators = new HashMap<>();
    CopyParameterFactory weight = new CopyParameterFactory(getMatrix(layer.getParam(DefaultParamInitializer.WEIGHT_KEY)));
    configurators.put(WeightLayer.WEIGHT_KEY, new ParameterConfigurator(0.01F, 0.05F, weight));
    CopyParameterFactory bias = new CopyParameterFactory(getMatrix(layer.getParam(DefaultParamInitializer.BIAS_KEY)));
    configurators.put(WeightLayer.BIAS_KEY, new ParameterConfigurator(0.01F, 0.05F, bias));
    MathCache factory = new DenseCache();
    return new WeightLayer(2, 1, factory, configurators, new SigmoidActivationFunction());
}
 

@Override
protected List<KeyValue<String, String>> getGradients() {
    List<KeyValue<String, String>> gradients = new LinkedList<>();
    gradients.add(new KeyValue<>(DefaultParamInitializer.WEIGHT_KEY, EmbedLayer.WEIGHT_KEY));
    gradients.add(new KeyValue<>(DefaultParamInitializer.BIAS_KEY, EmbedLayer.BIAS_KEY));
    return gradients;
}
 

@Override
protected Layer getNewFunction(AbstractLayer<?> layer) {
    Map<String, ParameterConfigurator> configurators = new HashMap<>();
    CopyParameterFactory weight = new CopyParameterFactory(getMatrix(layer.getParam(DefaultParamInitializer.WEIGHT_KEY)));
    configurators.put(WeightLayer.WEIGHT_KEY, new ParameterConfigurator(0.01F, 0.05F, weight));
    CopyParameterFactory bias = new CopyParameterFactory(getMatrix(layer.getParam(DefaultParamInitializer.BIAS_KEY)));
    configurators.put(WeightLayer.BIAS_KEY, new ParameterConfigurator(0.01F, 0.05F, bias));
    MathCache factory = new DenseCache();
    return new EmbedLayer(5, 2, factory, configurators, new SigmoidActivationFunction());
}
 

@Test
public void testEmbeddingLayerSetWeightsMaskZero() throws Exception {
    //GIVEN keras embedding with mask zero true
    KerasEmbedding embedding = buildEmbeddingLayer(conf1, keras1, true);
    //WHEN
    embedding.setWeights(Collections.singletonMap(conf1.getLAYER_FIELD_EMBEDDING_WEIGHTS(), Nd4j.ones(INPUT_SHAPE)));
    //THEN first row is set to zeros
    INDArray weights = embedding.getWeights().get(DefaultParamInitializer.WEIGHT_KEY);
    assertEquals(embedding.getWeights().get(DefaultParamInitializer.WEIGHT_KEY).columns(),INPUT_SHAPE[1]);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray input = this.input;
    if (input.rank() != 3)
        throw new UnsupportedOperationException(
                        "Input must be rank 3. Got input with rank " + input.rank() + " " + layerId());
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);

    applyDropOutIfNecessary(training, workspaceMgr);
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
    }
    INDArray input2d = TimeSeriesUtils.reshape3dTo2d(input.castTo(W.dataType()), workspaceMgr, ArrayType.FF_WORKING_MEM);

    INDArray act2d = layerConf().getActivationFn().getActivation(input2d.mmul(W).addiRowVector(b), training);
    if (maskArray != null) {
        if(!maskArray.isColumnVectorOrScalar() || Arrays.equals(maskArray.shape(), act2d.shape())){
            //Per output masking
            act2d.muli(maskArray.castTo(act2d.dataType()));
        } else {
            //Per time step masking
            act2d.muliColumnVector(maskArray.castTo(act2d.dataType()));
        }
    }

    INDArray ret = TimeSeriesUtils.reshape2dTo3d(act2d, input.size(0), workspaceMgr, ArrayType.ACTIVATIONS);
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        ret = ret.permute(0, 2, 1);
    }
    return ret;
}