org.deeplearning4j.nn.conf.layers.EmbeddingLayer Java Examples

private ComputationGraph getOldFunction() {
    NeuralNetConfiguration.Builder netBuilder = new NeuralNetConfiguration.Builder();
    // 设置随机种子
    netBuilder.seed(6);
    netBuilder.setL1(l1Regularization);
    netBuilder.setL1Bias(l1Regularization);
    netBuilder.setL2(l2Regularization);
    netBuilder.setL2Bias(l2Regularization);
    netBuilder.weightInit(WeightInit.XAVIER_UNIFORM);
    netBuilder.updater(new Sgd(learnRatio)).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);

    GraphBuilder graphBuilder = netBuilder.graphBuilder();
    graphBuilder.addInputs("leftInput", "rightInput");
    graphBuilder.addLayer("leftEmbed", new EmbeddingLayer.Builder().nIn(5).nOut(5).hasBias(true).activation(Activation.IDENTITY).build(), "leftInput");
    graphBuilder.addLayer("rightEmbed", new EmbeddingLayer.Builder().nIn(5).nOut(5).hasBias(true).activation(Activation.IDENTITY).build(), "rightInput");
    graphBuilder.addVertex("embed", new MergeVertex(), "leftEmbed", "rightEmbed");
    graphBuilder.addLayer("output", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(10).nOut(1).build(), "embed");
    graphBuilder.setOutputs("output");

    ComputationGraphConfiguration configuration = graphBuilder.build();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();
    return graph;
}
 

@Override
protected AbstractLayer<?> getOldFunction() {
    NeuralNetConfiguration neuralNetConfiguration = new NeuralNetConfiguration();
    EmbeddingLayer layerConfiguration = new EmbeddingLayer();
    layerConfiguration.setWeightInit(WeightInit.UNIFORM);
    layerConfiguration.setNIn(5);
    layerConfiguration.setNOut(2);
    layerConfiguration.setActivationFn(new ActivationSigmoid());
    layerConfiguration.setL1(0.01D);
    layerConfiguration.setL1Bias(0.01D);
    layerConfiguration.setL2(0.05D);
    layerConfiguration.setL2Bias(0.05D);
    neuralNetConfiguration.setLayer(layerConfiguration);
    AbstractLayer<?> layer = AbstractLayer.class.cast(layerConfiguration.instantiate(neuralNetConfiguration, null, 0, Nd4j.zeros(12), true));
    layer.setBackpropGradientsViewArray(Nd4j.zeros(12));
    return layer;
}
 

@Test
public void testEmbeddingLayerConfig() {

    for (boolean hasBias : new boolean[]{true, false}) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(0, new EmbeddingLayer.Builder().hasBias(hasBias).nIn(10).nOut(5).build())
                .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l0 = net.getLayer(0);

        assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingLayer.class, l0.getClass());
        assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn());
        assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut());

        INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY);
        INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY);
        assertArrayEquals(new long[]{10, 5}, weights.shape());
        if (hasBias) {
            assertArrayEquals(new long[]{1, 5}, bias.shape());
        }
    }
}
 

@Test
public void testEmbeddingDefaultActivation(){

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new EmbeddingLayer.Builder().nIn(10).nOut(10).build())
            .layer(new EmbeddingSequenceLayer.Builder().nIn(10).nOut(10).build())
            .build();

    EmbeddingLayer l = (EmbeddingLayer) conf.getConf(0).getLayer();
    assertEquals(new ActivationIdentity(), l.getActivationFn());

    EmbeddingSequenceLayer l2 = (EmbeddingSequenceLayer) conf.getConf(1).getLayer();
    assertEquals(new ActivationIdentity(), l2.getActivationFn());

}
 

/**
 * Check if the given layers are compatible for sequences (Only allow embedding and RNN for now)
 *
 * @param layer Layers to check
 * @return True if compatible
 */
protected boolean isSequenceCompatibleLayer(Layer layer) {
  return layer.getBackend() instanceof EmbeddingLayer
      || layer.getBackend() instanceof AbstractLSTM
      || layer.getBackend() instanceof RnnOutputLayer
      || layer.getBackend() instanceof GlobalPoolingLayer;
}
 

/**
 * Check if the given layers are compatible for sequences (Only allow embedding and RNN for now)
 *
 * @param layer Layers to check
 * @return True if compatible
 */
protected boolean isSequenceCompatibleLayer(Layer layer) {
  return layer.getBackend() instanceof EmbeddingLayer
      || layer.getBackend() instanceof AbstractLSTM
      || layer.getBackend() instanceof RnnOutputLayer
      || layer.getBackend() instanceof GlobalPoolingLayer;
}
 

@Test
public void testStackVertexEmbedding() {
    Nd4j.getRandom().setSeed(12345);
    GraphVertex unstack = new StackVertex(null, "", -1, Nd4j.dataType());

    INDArray in1 = Nd4j.zeros(5, 1);
    INDArray in2 = Nd4j.zeros(5, 1);
    for (int i = 0; i < 5; i++) {
        in1.putScalar(i, 0, i);
        in2.putScalar(i, 0, i);
    }

    INDArray l = Nd4j.rand(5, 5);
    MultiDataSet ds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[] {in1, in2}, new INDArray[] {l, l},
                    null, null);


    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in1", "in2")
                    .addVertex("stack", new org.deeplearning4j.nn.conf.graph.StackVertex(), "in1", "in2")
                    .addLayer("1", new EmbeddingLayer.Builder().nIn(5).nOut(5).build(), "stack")
                    .addVertex("unstack1", new org.deeplearning4j.nn.conf.graph.UnstackVertex(0, 2), "1")
                    .addVertex("unstack2", new org.deeplearning4j.nn.conf.graph.UnstackVertex(0, 2), "1")
                    .addLayer("out1", new OutputLayer.Builder().activation(Activation.TANH)
                                    .lossFunction(LossFunctions.LossFunction.L2).nIn(5).nOut(5).build(), "unstack1")
                    .addLayer("out2", new OutputLayer.Builder().activation(Activation.TANH)
                                    .lossFunction(LossFunctions.LossFunction.L2).nIn(5).nOut(5).build(), "unstack2")
                    .setOutputs("out1", "out2").build();

    ComputationGraph g = new ComputationGraph(conf);
    g.init();

    g.feedForward(new INDArray[] {in1, in2}, false);

    g.fit(ds);

}
 

@Override
public EmbeddingLayer getValue(double[] values) {
    //Using the builder here, to get default options
    EmbeddingLayer.Builder b = new EmbeddingLayer.Builder();
    setLayerOptionsBuilder(b, values);
    return b.build();
}
 

/**
 * 获取计算图配置
 * 
 * @param dimensionSizes
 * @return
 */
protected ComputationGraphConfiguration getComputationGraphConfiguration(int[] dimensionSizes) {
    NeuralNetConfiguration.Builder netBuilder = new NeuralNetConfiguration.Builder();
    // 设置随机种子
    netBuilder.seed(6);
    netBuilder.weightInit(WeightInit.XAVIER_UNIFORM);
    netBuilder.updater(new Sgd(learnRatio)).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
    netBuilder.l1(weightRegularization);

    GraphBuilder graphBuilder = netBuilder.graphBuilder();

    // 构建离散域(SparseField)节点
    String[] inputVertexNames = new String[dimensionSizes.length];
    int[] inputVertexSizes = new int[dimensionSizes.length];
    for (int fieldIndex = 0; fieldIndex < dimensionSizes.length; fieldIndex++) {
        inputVertexNames[fieldIndex] = "SparseField" + fieldIndex;
        // 每个离散特征的输入数
        inputVertexSizes[fieldIndex] = dimensionSizes[fieldIndex];
    }
    graphBuilder.addInputs(inputVertexNames);

    // 构建Embed节点
    // TODO 应该调整为配置项.
    int numberOfFactors = 10;
    // TODO Embed只支持输入的column为1.
    String[] embedVertexNames = new String[dimensionSizes.length];
    for (int fieldIndex = 0; fieldIndex < dimensionSizes.length; fieldIndex++) {
        embedVertexNames[fieldIndex] = "Embed" + fieldIndex;
        graphBuilder.addLayer(embedVertexNames[fieldIndex], new EmbeddingLayer.Builder().nIn(inputVertexSizes[fieldIndex]).nOut(numberOfFactors).activation(Activation.IDENTITY).build(), inputVertexNames[fieldIndex]);
    }

    // 构建因子分解机部分
    // 构建FM Plus节点(实际就是FM的输入)
    numberOfFeatures = 0;
    for (int fieldIndex = 0; fieldIndex < dimensionSizes.length; fieldIndex++) {
        numberOfFeatures += inputVertexSizes[fieldIndex];
    }
    // TODO 注意,由于EmbedLayer不支持与其它Layer共享输入,所以FM Plus节点构建自己的One Hot输入.
    graphBuilder.addInputs("FMInputs");
    graphBuilder.addLayer("FMPlus", new DeepFMInputConfiguration.Builder(dimensionSizes).nOut(1).activation(Activation.IDENTITY).build(), "FMInputs");

    // 构建FM Product节点
    // 注意:节点数量是(n*(n-1)/2)),n为Embed节点数量
    String[] productVertexNames = new String[dimensionSizes.length * (dimensionSizes.length - 1) / 2];
    int productIndex = 0;
    for (int outterFieldIndex = 0; outterFieldIndex < dimensionSizes.length; outterFieldIndex++) {
        for (int innerFieldIndex = outterFieldIndex + 1; innerFieldIndex < dimensionSizes.length; innerFieldIndex++) {
            productVertexNames[productIndex] = "FMProduct" + outterFieldIndex + ":" + innerFieldIndex;
            String left = embedVertexNames[outterFieldIndex];
            String right = embedVertexNames[innerFieldIndex];
            graphBuilder.addVertex(productVertexNames[productIndex], new DeepFMProductConfiguration(), left, right);
            productIndex++;
        }
    }

    // 构建FM Sum节点(实际就是FM的输出)
    String[] names = new String[productVertexNames.length + 1];
    System.arraycopy(productVertexNames, 0, names, 0, productVertexNames.length);
    names[names.length - 1] = "FMPlus";
    graphBuilder.addVertex("FMOutput", new DeepFMSumConfiguration(), names);

    // 构建多层网络部分
    // 构建Net Input节点
    // TODO 调整为支持输入(连续域)Dense Field.
    // TODO 应该调整为配置项.
    int numberOfHiddens = 100;
    graphBuilder.addLayer("NetInput", new DenseLayer.Builder().nIn(dimensionSizes.length * numberOfFactors).nOut(numberOfHiddens).activation(Activation.LEAKYRELU).build(), embedVertexNames);

    // TODO 应该调整为配置项.
    int numberOfLayers = 5;
    String currentLayer = "NetInput";
    for (int layerIndex = 0; layerIndex < numberOfLayers; layerIndex++) {
        graphBuilder.addLayer("NetHidden" + layerIndex, new DenseLayer.Builder().nIn(numberOfHiddens).nOut(numberOfHiddens).activation(Activation.LEAKYRELU).build(), currentLayer);
        currentLayer = "NetHidden" + layerIndex;
    }

    // 构建Net Output节点
    graphBuilder.addVertex("NetOutput", new DeepFMSumConfiguration(), currentLayer);

    // 构建Deep Output节点
    graphBuilder.addLayer("DeepOutput", new DeepFMOutputConfiguration.Builder(LossFunctions.LossFunction.XENT).activation(Activation.SIGMOID).nIn(2).nOut(1).build(), "FMOutput", "NetOutput");

    graphBuilder.setOutputs("DeepOutput");
    ComputationGraphConfiguration configuration = graphBuilder.build();
    return configuration;
}
 

@Test(timeout = 300000)
    public void testW2VEmbeddingLayerInit() throws Exception {
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);

        val inputFile = Resources.asFile("big/raw_sentences.txt");
        val iter = ParagraphVectorsTest.getIterator(isIntegrationTests(), inputFile);
//        val iter = new BasicLineIterator(inputFile);
        val t = new DefaultTokenizerFactory();
        t.setTokenPreProcessor(new CommonPreprocessor());

        Word2Vec vec = new Word2Vec.Builder()
                .minWordFrequency(1)
                .epochs(1)
                .layerSize(300)
                .limitVocabularySize(1) // Limit the vocab size to 2 words
                .windowSize(5)
                .allowParallelTokenization(true)
                .batchSize(512)
                .learningRate(0.025)
                .minLearningRate(0.0001)
                .negativeSample(0.0)
                .sampling(0.0)
                .useAdaGrad(false)
                .useHierarchicSoftmax(true)
                .iterations(1)
                .useUnknown(true) // Using UNK with limited vocab size causes the issue
                .seed(42)
                .iterate(iter)
                .workers(4)
                .tokenizerFactory(t).build();

        vec.fit();

        INDArray w = vec.lookupTable().getWeights();
        System.out.println(w);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345).list()
                .layer(new EmbeddingLayer.Builder().weightInit(vec).build())
                .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(w.size(1)).nOut(3).build())
                .layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                        .nOut(4).build())
                .build();

        final MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray w0 = net.getParam("0_W");
        assertEquals(w, w0);

        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ModelSerializer.writeModel(net, baos, true);
        byte[] bytes = baos.toByteArray();

        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
        final MultiLayerNetwork restored = ModelSerializer.restoreMultiLayerNetwork(bais, true);

        assertEquals(net.getLayerWiseConfigurations(), restored.getLayerWiseConfigurations());
        assertTrue(net.params().equalsWithEps(restored.params(), 2e-3));
    }
 

@Test
public void testEmbeddingForwardPass() {
    //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation
    // input with a DenseLayer

    int nClassesIn = 10;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
            .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build())
            .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build())
            .build();
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
            .layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build())
            .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net.init();
    net2.init();

    net2.setParams(net.params().dup());

    int batchSize = 3;
    INDArray inEmbedding = Nd4j.create(batchSize, 1);
    INDArray inOneHot = Nd4j.create(batchSize, nClassesIn);

    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        int classIdx = r.nextInt(nClassesIn);
        inEmbedding.putScalar(i, classIdx);
        inOneHot.putScalar(new int[]{i, classIdx}, 1.0);
    }

    List<INDArray> activationsEmbedding = net.feedForward(inEmbedding, false);
    List<INDArray> activationsDense = net2.feedForward(inOneHot, false);
    for (int i = 1; i < 3; i++) {
        INDArray actE = activationsEmbedding.get(i);
        INDArray actD = activationsDense.get(i);
        assertEquals(actE, actD);
    }
}
 

@Test
public void testEmbeddingBackwardPass() {
    //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation
    // input with a DenseLayer

    int nClassesIn = 10;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
            .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()).layer(1,
                    new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4)
                            .activation(Activation.SOFTMAX).build())
            .build();
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH)
            .weightInit(WeightInit.XAVIER).list()
            .layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4)
                            .activation(Activation.SOFTMAX).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net.init();
    net2.init();

    net2.setParams(net.params().dup());

    int batchSize = 3;
    INDArray inEmbedding = Nd4j.create(batchSize, 1);
    INDArray inOneHot = Nd4j.create(batchSize, nClassesIn);
    INDArray outLabels = Nd4j.create(batchSize, 4);

    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        int classIdx = r.nextInt(nClassesIn);
        inEmbedding.putScalar(i, classIdx);
        inOneHot.putScalar(new int[]{i, classIdx}, 1.0);

        int labelIdx = r.nextInt(4);
        outLabels.putScalar(new int[]{i, labelIdx}, 1.0);
    }

    net.setInput(inEmbedding);
    net2.setInput(inOneHot);
    net.setLabels(outLabels);
    net2.setLabels(outLabels);

    net.computeGradientAndScore();
    net2.computeGradientAndScore();

    assertEquals(net2.score(), net.score(), 1e-6);

    Map<String, INDArray> gradient = net.gradient().gradientForVariable();
    Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
    assertEquals(gradient.size(), gradient2.size());

    for (String s : gradient.keySet()) {
        assertEquals(gradient2.get(s), gradient.get(s));
    }
}
 

@Test
    public void testEmbeddingLayerRNN() {

        int nClassesIn = 10;

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH)
                .dataType(DataType.DOUBLE)
                .list()
                .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build())
                .layer(1, new GravesLSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build())
                .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4)
                        .activation(Activation.SOFTMAX).build())
                .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                .inputPreProcessor(1, new FeedForwardToRnnPreProcessor())
                .build();
        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH)
                .weightInit(WeightInit.XAVIER)
                .dataType(DataType.DOUBLE)
                .list()
                .layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build())
                .layer(1, new GravesLSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build())
                .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4)
                        .activation(Activation.SOFTMAX).build())
                .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                .inputPreProcessor(1, new FeedForwardToRnnPreProcessor())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net.init();
        net2.init();

        net2.setParams(net.params().dup());

        int batchSize = 3;
        int timeSeriesLength = 8;
        INDArray inEmbedding = Nd4j.create(batchSize, 1, timeSeriesLength);
        INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, timeSeriesLength);
        INDArray outLabels = Nd4j.create(batchSize, 4, timeSeriesLength);

        Random r = new Random(12345);
        for (int i = 0; i < batchSize; i++) {
            for (int j = 0; j < timeSeriesLength; j++) {
                int classIdx = r.nextInt(nClassesIn);
                inEmbedding.putScalar(new int[]{i, 0, j}, classIdx);
                inOneHot.putScalar(new int[]{i, classIdx, j}, 1.0);

                int labelIdx = r.nextInt(4);
                outLabels.putScalar(new int[]{i, labelIdx, j}, 1.0);
            }
        }

        net.setInput(inEmbedding);
        net2.setInput(inOneHot);
        net.setLabels(outLabels);
        net2.setLabels(outLabels);

        net.computeGradientAndScore();
        net2.computeGradientAndScore();

//        System.out.println(net.score() + "\t" + net2.score());
        assertEquals(net2.score(), net.score(), 1e-5);

        Map<String, INDArray> gradient = net.gradient().gradientForVariable();
        Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
        assertEquals(gradient.size(), gradient2.size());

        for (String s : gradient.keySet()) {
            assertEquals(gradient2.get(s), gradient.get(s));
        }

    }
 

@Test
    public void testEmbeddingLayerWithMasking() {
        //Idea: have masking on the input with an embedding and dense layers on input
        //Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked

        int[] miniBatchSizes = {1, 2, 5};
        int nIn = 2;
        Random r = new Random(12345);

        int numInputClasses = 10;
        int timeSeriesLength = 5;

        for (DataType maskDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) {
            for (int nExamples : miniBatchSizes) {
                Nd4j.getRandom().setSeed(12345);

                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .updater(new Sgd(0.1)).seed(12345).list()
                        .layer(0, new EmbeddingLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses)
                                .nOut(5).build())
                        .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build())
                        .layer(2, new GravesLSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build())
                        .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                                .nOut(4).build())
                        .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                        .inputPreProcessor(2, new FeedForwardToRnnPreProcessor()).build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .updater(new Sgd(0.1)).seed(12345).list()
                        .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5)
                                .build())
                        .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build())
                        .layer(2, new GravesLSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build())
                        .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                                .nOut(4).build())
                        .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                        .inputPreProcessor(2, new FeedForwardToRnnPreProcessor()).build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                net2.setParams(net.params().dup());

                INDArray inEmbedding = Nd4j.zeros(nExamples, 1, timeSeriesLength);
                INDArray inDense = Nd4j.zeros(nExamples, numInputClasses, timeSeriesLength);

                INDArray labels = Nd4j.zeros(nExamples, 4, timeSeriesLength);

                for (int i = 0; i < nExamples; i++) {
                    for (int j = 0; j < timeSeriesLength; j++) {
                        int inIdx = r.nextInt(numInputClasses);
                        inEmbedding.putScalar(new int[]{i, 0, j}, inIdx);
                        inDense.putScalar(new int[]{i, inIdx, j}, 1.0);

                        int outIdx = r.nextInt(4);
                        labels.putScalar(new int[]{i, outIdx, j}, 1.0);
                    }
                }

                INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength);
                for (int i = 0; i < nExamples; i++) {
                    for (int j = 0; j < timeSeriesLength; j++) {
                        inputMask.putScalar(new int[]{i, j}, (r.nextBoolean() ? 1.0 : 0.0));
                    }
                }

                net.setLayerMaskArrays(inputMask, null);
                net2.setLayerMaskArrays(inputMask, null);
                List<INDArray> actEmbedding = net.feedForward(inEmbedding, false);
                List<INDArray> actDense = net2.feedForward(inDense, false);
                for (int i = 1; i < actEmbedding.size(); i++) {
                    assertEquals(actDense.get(i), actEmbedding.get(i));
                }

                net.setLabels(labels);
                net2.setLabels(labels);
                net.computeGradientAndScore();
                net2.computeGradientAndScore();

//                System.out.println(net.score() + "\t" + net2.score());
                assertEquals(net2.score(), net.score(), 1e-5);

                Map<String, INDArray> gradients = net.gradient().gradientForVariable();
                Map<String, INDArray> gradients2 = net2.gradient().gradientForVariable();
                assertEquals(gradients.keySet(), gradients2.keySet());
                for (String s : gradients.keySet()) {
                    assertEquals(gradients2.get(s), gradients.get(s));
                }
            }
        }
    }
 

@Test
public void testEmbeddingWeightInit(){
    // https://github.com/eclipse/deeplearning4j/issues/8663
    //The embedding layer weight initialization should be independent of the vocabulary size (nIn setting)

    for(WeightInit wi : new WeightInit[]{WeightInit.XAVIER, WeightInit.RELU, WeightInit.XAVIER_UNIFORM, WeightInit.LECUN_NORMAL}) {

        for (boolean seq : new boolean[]{false, true}) {

            Nd4j.getRandom().setSeed(12345);
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .seed(12345)
                    .list()
                    .layer(seq ?
                            new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100).nOut(100).build() :
                            new EmbeddingLayer.Builder().weightInit(wi).nIn(100).nOut(100).build())
                    .build();
            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            Nd4j.getRandom().setSeed(12345);
            MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                    .seed(12345)
                    .list()
                    .layer(seq ?
                            new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100).nOut(100).build() :
                            new EmbeddingLayer.Builder().weightInit(wi).nIn(100).nOut(100).build())
                    .build();
            MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
            net2.init();

            Nd4j.getRandom().setSeed(12345);
            MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder()
                    .seed(12345)
                    .list()
                    .layer(seq ?
                            new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100000).nOut(100).build() :
                            new EmbeddingLayer.Builder().weightInit(wi).nIn(100000).nOut(100).build())
                    .build();
            MultiLayerNetwork net3 = new MultiLayerNetwork(conf3);
            net3.init();

            INDArray p1 = net.params();
            INDArray p2 = net2.params();
            INDArray p3 = net3.params();
            boolean eq = p1.equalsWithEps(p2, 1e-4);
            String str = (seq ? "EmbeddingSequenceLayer" : "EmbeddingLayer") + " - " + wi;
            assertTrue(str + " p1/p2 params not equal", eq);

            double m1 = p1.meanNumber().doubleValue();
            double s1 = p1.stdNumber().doubleValue();

            double m3 = p3.meanNumber().doubleValue();
            double s3 = p3.stdNumber().doubleValue();



            assertEquals(str, m1, m3, 0.1);
            assertEquals(str, s1, s3, 0.1);

            double re = relErr(s1, s3);
            assertTrue(str + " - " + re, re < 0.05);
        }
    }

}
 

protected void setLayerOptionsBuilder(EmbeddingLayer.Builder builder, double[] values) {
    super.setLayerOptionsBuilder(builder, values);
    if(hasBias != null)
        builder.hasBias(hasBias.getValue(values));
}