org.nd4j.linalg.learning.config.Adam Java Examples

public static MultiLayerNetwork getTestNet() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1e-3))
            .list()
            .layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(3, 3).activation(Activation.TANH).build())
            .layer(new BatchNormalization.Builder().nOut(5).build())
            .layer(new SubsamplingLayer.Builder().build())
            .layer(new DenseLayer.Builder().nOut(10).activation(Activation.RELU).build())
            .layer(new OutputLayer.Builder().nOut(10).build())
            .setInputType(InputType.convolutional(28, 28, 1))
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    return net;
}
 

@Override
public Object getConfiguration() throws Exception {
    return new NeuralNetConfiguration.Builder()
            .dataType(DataType.FLOAT)
            .seed(12345)
            .updater(new Adam(5e-2))
            .l1(1e-3).l2(1e-3)
            .list()
            .layer(0, new Bidirectional(new LSTM.Builder().activation(Activation.TANH).nOut(10).build()))
            .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build())
            .layer(new OutputLayer.Builder().nOut(6)
                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.recurrent(1))
            .build();
}
 

public static SameDiff getModel(){
    Nd4j.getRandom().setSeed(12345);
    SameDiff sd = SameDiff.create();
    SDVariable in = sd.placeHolder("in", DataType.FLOAT, -1, 4);
    SDVariable label = sd.placeHolder("label", DataType.FLOAT, -1, 3);
    SDVariable w = sd.var("W", Nd4j.rand(DataType.FLOAT, 4, 3));
    SDVariable b = sd.var("b", DataType.FLOAT, 3);

    SDVariable mmul = in.mmul(w).add(b);
    SDVariable softmax = sd.nn().softmax(mmul);
    SDVariable loss = sd.loss().logLoss("loss", label, softmax);

    sd.setTrainingConfig(TrainingConfig.builder()
            .dataSetFeatureMapping("in")
            .dataSetLabelMapping("label")
            .updater(new Adam(1e-2))
            .weightDecay(1e-2, true)
            .build());

    return sd;
}
 

@Override
public IUpdater getValue(double[] parameterValues) {
    double lr = learningRate == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : learningRate.getValue(parameterValues);
    ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues);
    double b1 = beta1 == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : beta1.getValue(parameterValues);
    double b2 = beta2 == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : beta2.getValue(parameterValues);
    double eps = epsilon == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : epsilon.getValue(parameterValues);
    if(lrS == null){
        return new Adam(lr, b1, b2, eps);
    } else {
        Adam a = new Adam(lrS);
        a.setBeta1(b1);
        a.setBeta2(b2);
        a.setEpsilon(eps);
        return a;
    }
}
 

@Test
public void testUpdaterAdamParamsLayerwiseOverride() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1.0, 0.5, 0.5, 1e-8))
            .list()
            .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build())
                    .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Adam(1.0, 0.6, 0.7, 1e-8)).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta1(), 0.0);
    assertEquals(0.6, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta1(), 0.0);
    assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta2(), 0.0);
    assertEquals(0.7, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta2(), 0.0);
}
 

@Ignore
    @Test
    public void testFailureIter5() throws Exception {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .updater(new Adam(1e-4))
                .list()
                .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        net.setListeners(new FailureTestingListener(
//                FailureTestingListener.FailureMode.OOM,
                FailureTestingListener.FailureMode.SYSTEM_EXIT_1,
                new FailureTestingListener.IterationEpochTrigger(false, 10)));

        DataSetIterator iter = new IrisDataSetIterator(5,150);

        net.fit(iter);
    }
 

@Override
public Object getConfiguration() throws Exception {
    return new NeuralNetConfiguration.Builder()
            .dataType(DataType.FLOAT)
            .seed(12345)
            .updater(new Adam(5e-2))
            .l1(1e-3).l2(1e-3)
            .list()
            .layer(0, new LSTM.Builder().activation(Activation.TANH).nOut(10).build())
            .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build())
            .layer(new OutputLayer.Builder().nOut(6)
                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.recurrent(1))
            .build();
}
 

@Test
public void testCustomListener() {
    SameDiff sd = SameDiff.create();
    SDVariable in = sd.placeHolder("input", DataType.FLOAT, -1, 4);
    SDVariable label = sd.placeHolder("label", DataType.FLOAT, -1, 3);
    SDVariable w = sd.var("w", Nd4j.rand(DataType.FLOAT, 4, 3));
    SDVariable b = sd.var("b", Nd4j.rand(DataType.FLOAT, 3));
    SDVariable z = sd.nn().linear("z", in, w, b);
    SDVariable out = sd.nn().softmax("out", z, 1);
    SDVariable loss = sd.loss().softmaxCrossEntropy("loss", label, out, null);

    //Create and set the training configuration
    double learningRate = 1e-3;
    TrainingConfig config = new TrainingConfig.Builder()
            .l2(1e-4)                               //L2 regularization
            .updater(new Adam(learningRate))        //Adam optimizer with specified learning rate
            .dataSetFeatureMapping("input")         //DataSet features array should be associated with variable "input"
            .dataSetLabelMapping("label")           //DataSet label array should be associated with variable "label
            .addEvaluations(false,"out",0,new Evaluation())
            .build();
    sd.setTrainingConfig(config);

    CustomListener listener = new CustomListener();
    Map<String,INDArray> m = sd.output()
            .data(new IrisDataSetIterator(150, 150))
            .output("out")
            .listeners(listener)
            .exec();

    assertEquals(1, m.size());
    assertTrue(m.containsKey("out"));
    assertNotNull(listener.z);
    assertNotNull(listener.out);

}
 

public DQN buildDQN(int shapeInputs[], int numOutputs) {

        if (shapeInputs.length == 1)
            throw new AssertionError("Impossible to apply convolutional layer on a shape == 1");


        NeuralNetConfiguration.ListBuilder confB = new NeuralNetConfiguration.Builder().seed(Constants.NEURAL_NET_SEED)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .l2(conf.getL2())
                        .updater(conf.getUpdater() != null ? conf.getUpdater() : new Adam())
                        .weightInit(WeightInit.XAVIER).l2(conf.getL2()).list()
                        .layer(0, new ConvolutionLayer.Builder(8, 8).nIn(shapeInputs[0]).nOut(16).stride(4, 4)
                                        .activation(Activation.RELU).build());


        confB.layer(1, new ConvolutionLayer.Builder(4, 4).nOut(32).stride(2, 2).activation(Activation.RELU).build());

        confB.layer(2, new DenseLayer.Builder().nOut(256).activation(Activation.RELU).build());

        confB.layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nOut(numOutputs)
                        .build());

        confB.setInputType(InputType.convolutional(shapeInputs[1], shapeInputs[2], shapeInputs[0]));
        MultiLayerConfiguration mlnconf = confB.build();
        MultiLayerNetwork model = new MultiLayerNetwork(mlnconf);
        model.init();
        if (conf.getListeners() != null) {
            model.setListeners(conf.getListeners());
        } else {
            model.setListeners(new ScoreIterationListener(Constants.NEURAL_NET_ITERATION_LISTENER));
        }

        return new DQN(model);
    }
 

@Test
public void testBiasLr() {
    //setup the network
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(1e-2))
                    .biasUpdater(new Adam(0.5)).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
                    .layer(2, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10)
                                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutional(28, 28, 1)).build();

    org.deeplearning4j.nn.conf.layers.BaseLayer l0 = (BaseLayer) conf.getConf(0).getLayer();
    org.deeplearning4j.nn.conf.layers.BaseLayer l1 = (BaseLayer) conf.getConf(1).getLayer();
    org.deeplearning4j.nn.conf.layers.BaseLayer l2 = (BaseLayer) conf.getConf(2).getLayer();
    org.deeplearning4j.nn.conf.layers.BaseLayer l3 = (BaseLayer) conf.getConf(3).getLayer();

    assertEquals(0.5, ((Adam)l0.getUpdaterByParam("b")).getLearningRate(), 1e-6);
    assertEquals(1e-2, ((Adam)l0.getUpdaterByParam("W")).getLearningRate(), 1e-6);

    assertEquals(0.5, ((Adam)l1.getUpdaterByParam("b")).getLearningRate(), 1e-6);
    assertEquals(1e-2, ((Adam)l1.getUpdaterByParam("W")).getLearningRate(), 1e-6);

    assertEquals(0.5, ((Adam)l2.getUpdaterByParam("b")).getLearningRate(), 1e-6);
    assertEquals(1e-2, ((Adam)l2.getUpdaterByParam("W")).getLearningRate(), 1e-6);

    assertEquals(0.5, ((Adam)l3.getUpdaterByParam("b")).getLearningRate(), 1e-6);
    assertEquals(1e-2, ((Adam)l3.getUpdaterByParam("W")).getLearningRate(), 1e-6);
}
 

@Ignore
@Test
public void testFailureRandom_AND() throws Exception {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1e-4))
            .list()
            .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    String hostname = InetAddress.getLocalHost().getHostName();
    assertNotNull(hostname);
    assertFalse(hostname.isEmpty());

    net.setListeners(new FailureTestingListener(
            FailureTestingListener.FailureMode.ILLEGAL_STATE,
            new FailureTestingListener.And(
                    new FailureTestingListener.HostNameTrigger(hostname),
                    new FailureTestingListener.RandomProb(FailureTestingListener.CallType.ANY, 0.05))
    ));

    DataSetIterator iter = new IrisDataSetIterator(5,150);

    net.fit(iter);
}
 

/**
 * Creates a simple feed-forward {@link MultiLayerNetwork} that can be used as a
 * PLNet for dyad-ranking.
 *
 * @param numInputs
 *            The number of inputs to the network, i.e. the number of features
 *            of a dyad.
 * @return New {@link MultiLayerNetwork}
 */
private MultiLayerNetwork createNetwork(final int numInputs) {
	if (this.configuration.plNetHiddenNodes().isEmpty()) {
		throw new IllegalArgumentException("There must be at least one hidden layer in specified in the config file!");
	}
	ListBuilder configBuilder = new NeuralNetConfiguration.Builder().seed(this.configuration.plNetSeed())
			// Gradient descent updater: Adam
			.updater(new Adam(this.configuration.plNetLearningRate())).list();

	// Build hidden layers
	String activation = this.configuration.plNetActivationFunction();
	int inputsFirstHiddenLayer = this.configuration.plNetHiddenNodes().get(0);
	configBuilder.layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(inputsFirstHiddenLayer).weightInit(WeightInit.SIGMOID_UNIFORM).activation(Activation.fromString(activation)).hasBias(true).build());
	List<Integer> hiddenNodes = this.configuration.plNetHiddenNodes();

	for (int i = 0; i < hiddenNodes.size() - 1; i++) {
		int numIn = hiddenNodes.get(i);
		int numOut = hiddenNodes.get(i + 1);
		configBuilder.layer(i + 1, new DenseLayer.Builder().nIn(numIn).nOut(numOut).weightInit(WeightInit.SIGMOID_UNIFORM).activation(Activation.fromString(activation)).hasBias(true).build());
	}

	// Build output layer. Since we are using an external error for training,
	// this is a regular layer instead of an OutputLayer
	configBuilder.layer(hiddenNodes.size(), new DenseLayer.Builder().nIn(hiddenNodes.get(hiddenNodes.size() - 1)).nOut(1).weightInit(WeightInit.UNIFORM).activation(Activation.IDENTITY).hasBias(true).build());

	MultiLayerConfiguration multiLayerConfig = configBuilder.build();
	return new MultiLayerNetwork(multiLayerConfig);
}
 

public static MultiLayerNetwork createIrisMLN(){
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(0.01))
            .list()
            .layer(new DenseLayer.Builder().nOut(5).activation(Activation.RELU).build())
            .layer(new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .setInputType(InputType.feedForward(4))
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    return net;
}
 

private AdamUpdater createADAMUpdater() {
    AdamUpdater adamUpdater = new AdamUpdater(new Adam(LEARNING_RATE, BETA_MOMENTUM, BETA2_MOMENTUM, EPSILON));
    adamUpdater.setStateViewArray(Nd4j.zeros(1, 2 * CHANNELS * WIDTH * HEIGHT),
        new long[]{1, CHANNELS, HEIGHT, WIDTH}, 'c',
        true);
    return adamUpdater;
}
 

@Override
protected GradientUpdater<?> getOldFunction(long[] shape) {
    Adam configuration = new Adam();
    GradientUpdater<?> oldFunction = new AdamUpdater(configuration);
    int length = (int) (shape[0] * configuration.stateSize(shape[1]));
    INDArray view = Nd4j.zeros(length);
    oldFunction.setStateViewArray(view, shape, 'c', true);
    return oldFunction;
}
 

private AdamUpdater createADAMUpdater() {
    AdamUpdater adamUpdater = new AdamUpdater(new Adam(LEARNING_RATE, BETA_MOMENTUM, BETA2_MOMENTUM, EPSILON));
    adamUpdater.setStateViewArray(Nd4j.zeros(1, 2 * CHANNELS * WIDTH * HEIGHT),
            new int[]{1, CHANNELS, HEIGHT, WIDTH}, 'c',
            true);
    return adamUpdater;
}
 

@Test
    public void testTrainingEvalVarNotReqForLoss(){
        //If a variable is not required for the loss - normally it won't be calculated
        //But we want to make sure it IS calculated here - so we can perform evaluation on it

        SameDiff sd = SameDiff.create();
        SDVariable in = sd.placeHolder("in", DataType.FLOAT, -1, 4);
        SDVariable label = sd.placeHolder("label", DataType.FLOAT, -1, 3);
        SDVariable w = sd.var("w", Nd4j.rand(DataType.FLOAT, 4, 3));
        SDVariable z = in.mmul(w);
        SDVariable out = sd.nn.softmax("softmax", z);
        SDVariable loss = sd.loss.logLoss("loss", label, out);
        SDVariable notRequiredForLoss = sd.nn.softmax("notRequiredForLoss", z);

        sd.setTrainingConfig(TrainingConfig.builder()
                .updater(new Adam(0.001))
                .dataSetFeatureMapping("in")
                .dataSetLabelMapping("label")
                .trainEvaluation("notRequiredForLoss", 0, new Evaluation())
                .build());

//        sd.setListeners(new ScoreListener(1));

        DataSet ds = new DataSet(Nd4j.rand(DataType.FLOAT, 3, 4), Nd4j.createFromArray(new float[][]{{1,0,0}, {0,1,0}, {0,0,1}}));

        History h = sd.fit()
                .train(new SingletonDataSetIterator(ds), 4)
                .exec();

        List<Double> l = h.trainingEval(Evaluation.Metric.ACCURACY);
        assertEquals(4, l.size());
    }
 

@Test
public void testBatchNorm() throws Exception {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .updater(new Adam(1e-3))
            .activation(Activation.TANH)
            .list()
            .layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(2, 2).build())
            .layer(new BatchNormalization())
            .layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(2, 2).build())
            .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nOut(10).build())
            .setInputType(InputType.convolutionalFlat(28, 28, 1))
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DataSetIterator iter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, true, 12345), 10);

    net.fit(iter);

    MultiLayerNetwork net2 = new TransferLearning.Builder(net)
            .fineTuneConfiguration(FineTuneConfiguration.builder()
                    .updater(new AdaDelta())
                    .build())
            .removeOutputLayer()
            .addLayer(new BatchNormalization.Builder().nOut(3380).build())
            .addLayer(new OutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nIn(3380).nOut(10).build())
            .build();

    net2.fit(iter);
}
 

@Test
public void testConvertToVariable() {
    Nd4j.getRandom().setSeed(12345);

    SameDiff sd = SameDiff.create();
    SDVariable in = sd.placeHolder("in", DataType.FLOAT, 1, 3);
    SDVariable w = sd.constant("w", Nd4j.rand(DataType.FLOAT, 3, 4));
    SDVariable b = sd.var("b", Nd4j.rand(DataType.FLOAT, 1, 4));
    SDVariable mmul = in.mmul(w);
    SDVariable add = mmul.add(b);
    SDVariable tanh = sd.math().tanh(add);
    SDVariable loss = sd.variance(tanh, true);

    INDArray inArr = Nd4j.rand(DataType.FLOAT, 1, 3);
    in.setArray(inArr);

    TrainingConfig c = TrainingConfig.builder()
            .updater(new Adam(0.1))
            .weightDecay(0.01, true)
            .dataSetFeatureMapping("in")
            .skipBuilderValidation(true)
            .build();
    sd.setTrainingConfig(c);

    INDArray out = tanh.eval();
    sd.fit(new SingletonMultiDataSetIterator(new DataSet(inArr, null).toMultiDataSet()), 1);
    w.convertToVariable();

    INDArray out2 = tanh.eval();

    assertNotEquals(out, out2);
    assertEquals(VariableType.VARIABLE, w.getVariableType());
    assertEquals(VariableType.VARIABLE, b.getVariableType());
    assertEquals(VariableType.ARRAY, add.getVariableType());
    assertEquals(VariableType.ARRAY, tanh.getVariableType());

    //Sanity check on training:
    sd.fit(new SingletonMultiDataSetIterator(new DataSet(inArr, null).toMultiDataSet()), 1);
}
 

@Test
public void testVariableRenaming2() {

    SameDiff sd = SameDiff.create();
    SDVariable v1 = sd.placeHolder("x", DataType.FLOAT, 3, 4);
    SDVariable v2 = sd.var("y", Nd4j.rand(DataType.FLOAT, 4, 5));
    SDVariable v3 = v1.mmul("oldName", v2);
    SDVariable v4 = v3.std("out", false);

    INDArray out = sd.outputSingle(Collections.singletonMap("x", Nd4j.rand(DataType.FLOAT, 3, 4)), "out");

    sd.setTrainingConfig(TrainingConfig.builder()
            .updater(new Adam(1e-3))
            .dataSetFeatureMapping("x")
            .markLabelsUnused()
            .build());

    sd.fit(new DataSet(Nd4j.rand(DataType.FLOAT, 3, 4), null));
    v3.rename("newName");
    sd.fit(new DataSet(Nd4j.rand(DataType.FLOAT, 3, 4), null));
}
 

@Test
public void testTrainingConfigJson(){
    for(IEvaluation e : new IEvaluation[]{new Evaluation(), new RegressionEvaluation(), new EvaluationBinary(), new ROC(),
            new ROCMultiClass(), new ROCBinary(), new EvaluationCalibration()}) {
        TrainingConfig config = new TrainingConfig.Builder()
                .l2(1e-4)
                .updater(new Adam(0.1))
                .dataSetFeatureMapping("out").dataSetLabelMapping("label")
                .trainEvaluation("out", 0, e)
                .build();
        String json = config.toJson();
        TrainingConfig fromJson = TrainingConfig.fromJson(json);
        assertEquals(config, fromJson);
    }
}
 

@Test
public void testClearing() {
    for(WorkspaceMode wsm : WorkspaceMode.values()) {
        ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder()
                .updater(new Adam())
                .inferenceWorkspaceMode(wsm)
                .trainingWorkspaceMode(wsm)
                .graphBuilder()
                .addInputs("in")
                .setInputTypes(InputType.recurrent(200))
                .addLayer("embeddings", new EmbeddingLayer.Builder().nIn(200).nOut(50).build(), "in")
                .addLayer("a", new GravesLSTM.Builder().nOut(300).activation(Activation.HARDTANH).build(), "embeddings")
                .addVertex("b", new LastTimeStepVertex("in"), "a")
                .addLayer("c", new DenseLayer.Builder().nOut(300).activation(Activation.HARDTANH).build(), "b")
                .addLayer("output", new LossLayer.Builder().lossFunction(LossFunctions.LossFunction.COSINE_PROXIMITY).build(), "c")
                .setOutputs("output")
                .build();


        final ComputationGraph computationGraph = new ComputationGraph(config);
        computationGraph.init();
        computationGraph.setListeners(new ScoreIterationListener(3));

        WSTestDataSetIterator iterator = new WSTestDataSetIterator();
        computationGraph.fit(iterator);
    }
}
 

@Test
    public void testExecDebugListener(){

        SameDiff sd = SameDiff.create();
        SDVariable in = sd.placeHolder("in", DataType.FLOAT, -1, 3);
        SDVariable label = sd.placeHolder("label", DataType.FLOAT, 1, 2);
        SDVariable w = sd.var("w", Nd4j.rand(DataType.FLOAT, 3, 2));
        SDVariable b = sd.var("b", Nd4j.rand(DataType.FLOAT, 1, 2));
        SDVariable sm = sd.nn.softmax("softmax", in.mmul(w).add(b));
        SDVariable loss = sd.loss.logLoss("loss", label, sm);

        INDArray i = Nd4j.rand(DataType.FLOAT, 1, 3);
        INDArray l = Nd4j.rand(DataType.FLOAT, 1, 2);

        sd.setTrainingConfig(TrainingConfig.builder()
                .dataSetFeatureMapping("in")
                .dataSetLabelMapping("label")
                .updater(new Adam(0.001))
                .build());

        for(ExecDebuggingListener.PrintMode pm : ExecDebuggingListener.PrintMode.values()){
            sd.setListeners(new ExecDebuggingListener(pm, -1, true));
//            sd.output(m, "softmax");
            sd.fit(new DataSet(i, l));

            System.out.println("\n\n\n");
        }

    }
 

public static MultiLayerNetwork getSimpleNet(){

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .updater(new Adam(0.01))
                .list()
                .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
                .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
                .layer(new OutputLayer.Builder().nIn(10).nOut(10).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        return net;
    }
 

@Test
public void testFromSvmLight() throws Exception {
    JavaRDD<LabeledPoint> data = MLUtils
                    .loadLibSVMFile(sc.sc(),
                                    new ClassPathResource("svmLight/iris_svmLight_0.txt").getTempFileFromArchive()
                                                    .getAbsolutePath())
                    .toJavaRDD().map(new TestFn());

    MultiLayerConfiguration conf =
                    new NeuralNetConfiguration.Builder().seed(123)
                                    .updater(new Adam(1e-6))
                            .weightInit(WeightInit.XAVIER)
                            .list()
                            .layer(new BatchNormalization.Builder().nIn(4).nOut(4).build())
                            .layer(new DenseLayer.Builder().nIn(4).nOut(32).activation(Activation.RELU).build())
                            .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(32).nOut(3)
                                                                    .activation(Activation.SOFTMAX).build())
                                    .build();



    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    System.out.println("Initializing network");
    SparkDl4jMultiLayer master = new SparkDl4jMultiLayer(sc, getBasicConf(),
                    new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 5, 1, 0));

    master.fitLabeledPoint(data);
}
 

private static MultiLayerConfiguration getIrisMultiLayerConfig(Activation activationFunction, OptimizationAlgorithm optimizer) {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(optimizer)
                    .updater(new Adam(0.01)).seed(12345L).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(100).weightInit(WeightInit.XAVIER)
                                    .activation(activationFunction).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).nIn(100).nOut(3)
                                                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                                    .build())
                    .build();


    return conf;
}
 

@Test
public void testVae() throws Exception {

    File f = Resources.asFile("regression_testing/100b3/VaeMNISTAnomaly_100b3.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    VariationalAutoencoder l0 = (VariationalAutoencoder) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationLReLU(), l0.getActivationFn());
    assertEquals(32, l0.getNOut());
    assertArrayEquals(new int[]{256, 256}, l0.getEncoderLayerSizes());
    assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes());
            assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new WeightDecay(1e-4, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new Adam(1e-3), l0.getIUpdater());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100b3/VaeMNISTAnomaly_Output_100b3.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f2))){
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100b3/VaeMNISTAnomaly_Input_100b3.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f3))){
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

@Test
public void testVae() throws Exception {

    File f = Resources.asFile("regression_testing/100b6/VaeMNISTAnomaly_100b6.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    VariationalAutoencoder l0 = (VariationalAutoencoder) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationLReLU(), l0.getActivationFn());
    assertEquals(32, l0.getNOut());
    assertArrayEquals(new int[]{256, 256}, l0.getEncoderLayerSizes());
    assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes());
    assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0));
    assertEquals(new Adam(1e-3), l0.getIUpdater());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100b6/VaeMNISTAnomaly_Output_100b6.bin");
    try (DataInputStream dis = new DataInputStream(new FileInputStream(f2))) {
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100b6/VaeMNISTAnomaly_Input_100b6.bin");
    try (DataInputStream dis = new DataInputStream(new FileInputStream(f3))) {
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

@Test
public void testVae() throws Exception {

    File f = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_100a.bin");
    MultiLayerNetwork net = MultiLayerNetwork.load(f, true);

    VariationalAutoencoder l0 = (VariationalAutoencoder) net.getLayer(0).conf().getLayer();
    assertEquals(new ActivationLReLU(), l0.getActivationFn());
    assertEquals(32, l0.getNOut());
    assertArrayEquals(new int[]{256, 256}, l0.getEncoderLayerSizes());
    assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes());
            assertEquals(new WeightInitXavier(), l0.getWeightInitFn());
    assertEquals(new WeightDecay(1e-4, false), TestUtils.getWeightDecayReg(l0));
    assertEquals(new Adam(0.05), l0.getIUpdater());

    INDArray outExp;
    File f2 = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_Output_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f2))){
        outExp = Nd4j.read(dis);
    }

    INDArray in;
    File f3 = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_Input_100a.bin");
    try(DataInputStream dis = new DataInputStream(new FileInputStream(f3))){
        in = Nd4j.read(dis);
    }

    INDArray outAct = net.output(in);

    assertEquals(outExp, outAct);
}
 

@Test
public void testVerticesAndMasking7027(){
    //https://github.com/deeplearning4j/deeplearning4j/issues/7027
    int inputSize = 300;
    int hiddenSize = 100;

    ComputationGraphConfiguration configuration = new NeuralNetConfiguration.Builder()
            .updater(new Adam())
            .graphBuilder()
            .addInputs("x_emb")
            .setInputTypes(InputType.recurrent(inputSize))
            .addLayer("agg_lstm", new Bidirectional(CONCAT, new LSTM.Builder().nIn(inputSize).nOut(hiddenSize/2).build()), "x_emb")
            .addLayer("agg_att", new DenseLayer.Builder().nIn(100).nOut(1).activation(Activation.SOFTMAX).build(), "agg_lstm")
            .addVertex("att", new PreprocessorVertex(new ComposableInputPreProcessor(new FeedForwardToRnnPreProcessor(), new PermutePreprocessor(new int[] {0,2,1}), new RnnToFeedForwardPreProcessor())), "agg_att")
            .addLayer("att_repeat", new RepeatVector.Builder(hiddenSize).build(),"att")
            .addVertex("att_trans", new PreprocessorVertex(new PermutePreprocessor(new int[] {0, 2, 1})), "att_repeat")
            .addVertex("mult", new ElementWiseVertex(ElementWiseVertex.Op.Product), "agg_lstm", "att_trans")
            .addLayer("sum", new GlobalPoolingLayer.Builder().build(), "mult")
            .addLayer("agg_out", new DenseLayer.Builder().nIn(100).nOut(6).activation(Activation.TANH).build(), "sum")
            .addLayer("output", new OutputLayer.Builder().nIn(6).nOut(6).lossFunction(LossFunctions.LossFunction.RECONSTRUCTION_CROSSENTROPY).build(), "agg_out")
            .setOutputs("output")
            .build();

    ComputationGraph net = new ComputationGraph(configuration);
    net.init();

    int dataSize = 10;
    int seqLen = 5;
    INDArray features = Nd4j.rand(new int[] {dataSize, inputSize, seqLen});
    INDArray labels = Nd4j.rand(new int[] {dataSize, 6});
    INDArray featuresMask = Nd4j.ones(dataSize, seqLen);
    INDArray labelsMask = Nd4j.ones(dataSize, 6);

    DataSet dataSet1 = new DataSet(features, labels);
    net.fit(dataSet1);
    DataSet dataSet2 = new DataSet(features, labels, featuresMask, labelsMask);
    net.fit(dataSet2);
}