org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction Java Examples
The following examples show how to use
org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: OutputLayerTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Test
public void testSetParams() {
NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder()
.optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT)
.updater(new Sgd(1e-1))
.layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(4).nOut(3)
.weightInit(WeightInit.ZERO).activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT).build())
.build();
long numParams = conf.getLayer().initializer().numParams(conf);
INDArray params = Nd4j.create(1, numParams);
OutputLayer l = (OutputLayer) conf.getLayer().instantiate(conf,
Collections.<TrainingListener>singletonList(new ScoreIterationListener(1)), 0, params, true, params.dataType());
params = l.params();
l.setParams(params);
assertEquals(params, l.params());
}
Example #2
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Test
public void testRnnTimeStepWithPreprocessor() {
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder()
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.list()
.layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(10)
.nOut(10).activation(Activation.TANH).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(10)
.nOut(10).activation(Activation.TANH).build())
.layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
.inputPreProcessor(0, new FeedForwardToRnnPreProcessor())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray in = Nd4j.rand(1, 10);
net.rnnTimeStep(in);
}
Example #3
| Source File: CenterLossOutputLayerTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
private ComputationGraph getGraph(int numLabels, double lambda) {
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1)).updater(new NoOp())
.graphBuilder().addInputs("input1")
.addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(),
"input1")
.addLayer("lossLayer", new CenterLossOutputLayer.Builder()
.lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(numLabels)
.lambda(lambda).activation(Activation.SOFTMAX).build(), "l1")
.setOutputs("lossLayer").build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
return graph;
}
Example #4
| Source File: NeuralNetworks.java From Machine-Learning-in-Java with MIT License | 6 votes |
|
private static MultiLayerNetwork softMaxRegression(int seed,
int iterations, int numRows, int numColumns, int outputNum) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.seed(seed)
.gradientNormalization(
GradientNormalization.ClipElementWiseAbsoluteValue)
.gradientNormalizationThreshold(1.0)
.iterations(iterations)
.momentum(0.5)
.momentumAfter(Collections.singletonMap(3, 0.9))
.optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT)
.list(1)
.layer(0,
new OutputLayer.Builder(
LossFunction.NEGATIVELOGLIKELIHOOD)
.activation("softmax")
.nIn(numColumns * numRows).nOut(outputNum)
.build()).pretrain(true).backprop(false)
.build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
return model;
}
Example #5
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Test
public void testRnnTimeStepWithPreprocessorGraph() {
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.graphBuilder().addInputs("in")
.addLayer("0", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(10).nOut(10)
.activation(Activation.TANH).build(), "in")
.addLayer("1", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(10).nOut(10)
.activation(Activation.TANH).build(), "0")
.addLayer("2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1")
.setOutputs("2").inputPreProcessor("0", new FeedForwardToRnnPreProcessor())
.build();
ComputationGraph net = new ComputationGraph(conf);
net.init();
INDArray in = Nd4j.rand(1, 10);
net.rnnTimeStep(in);
}
Example #6
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Test
public void testInvalidTPBTT() {
int nIn = 8;
int nOut = 25;
int nHiddenUnits = 17;
try {
new NeuralNetConfiguration.Builder()
.list()
.layer(new org.deeplearning4j.nn.conf.layers.LSTM.Builder().nIn(nIn).nOut(nHiddenUnits).build())
.layer(new GlobalPoolingLayer())
.layer(new OutputLayer.Builder(LossFunction.MSE).nIn(nHiddenUnits)
.nOut(nOut)
.activation(Activation.TANH).build())
.backpropType(BackpropType.TruncatedBPTT)
.build();
fail("Exception expected");
} catch (IllegalStateException e){
log.info(e.toString());
assertTrue(e.getMessage().contains("TBPTT") && e.getMessage().contains("validateTbpttConfig"));
}
}
Example #7
| Source File: TestMultiLayerSpace.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Test
public void testGlobalPoolingBasic() {
MultiLayerConfiguration expected = new NeuralNetConfiguration.Builder().updater(new Sgd(0.005)).seed(12345).list()
.layer(0, new GravesLSTM.Builder().nIn(10).nOut(10).build())
.layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.SUM).pnorm(7).build())
.layer(2, new OutputLayer.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(5).build())
.build();
MultiLayerSpace mls =
new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345)
.addLayer(new GravesLSTMLayerSpace.Builder().nIn(10).nOut(10).build())
.addLayer(new GlobalPoolingLayerSpace.Builder().poolingType(PoolingType.SUM)
.pNorm(7).build())
.addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT)
.activation(Activation.SOFTMAX)
.nIn(10).nOut(5).build())
.build();
int nParams = mls.numParameters();
assertEquals(0, nParams);
MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration();
assertEquals(expected, conf);
}
Example #8
| Source File: BackPropMLPTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
/** Very simple back-prop config set up for Iris.
* Learning Rate = 0.1
* No regularization, no Adagrad, no momentum etc. One iteration.
*/
private static MultiLayerConfiguration getIrisMLPSimpleConfig(int[] hiddenLayerSizes,
Activation activationFunction) {
NeuralNetConfiguration.ListBuilder lb = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1))
.seed(12345L).list();
for (int i = 0; i < hiddenLayerSizes.length; i++) {
int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]);
lb.layer(i, new DenseLayer.Builder().nIn(nIn).nOut(hiddenLayerSizes[i]).weightInit(WeightInit.XAVIER)
.activation(activationFunction).build());
}
lb.layer(hiddenLayerSizes.length,
new OutputLayer.Builder(LossFunction.MCXENT).nIn(hiddenLayerSizes[hiddenLayerSizes.length - 1])
.nOut(3).weightInit(WeightInit.XAVIER)
.activation(activationFunction.equals(Activation.IDENTITY) ? Activation.IDENTITY
: Activation.SOFTMAX)
.build());
return lb.build();
}
Example #9
| Source File: TestSetGetParameters.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testSetParametersRNN() {
//Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this.
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
.layer(0, new GravesLSTM.Builder().nIn(9).nOut(10)
.dist(new NormalDistribution(0, 1)).build())
.layer(1, new GravesLSTM.Builder().nIn(10).nOut(11)
.dist(new NormalDistribution(0, 1)).build())
.layer(2, new RnnOutputLayer.Builder(LossFunction.MSE)
.dist(new NormalDistribution(0, 1)).nIn(11).nOut(12).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray initParams = net.params().dup();
Map<String, INDArray> initParams2 = net.paramTable();
net.setParams(net.params());
INDArray initParamsAfter = net.params();
Map<String, INDArray> initParams2After = net.paramTable();
for (String s : initParams2.keySet()) {
assertTrue("Params differ: " + s, initParams2.get(s).equals(initParams2After.get(s)));
}
assertEquals(initParams, initParamsAfter);
//Now, try the other way: get(set(random))
INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape());
net.setParams(randomParams.dup());
assertEquals(net.params(), randomParams);
}
Example #10
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testTruncatedBPTTSimple() {
//Extremely simple test of the 'does it throw an exception' variety
int timeSeriesLength = 12;
int miniBatchSize = 7;
int nIn = 5;
int nOut = 4;
int nTimeSlices = 20;
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
.layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
.activation(Activation.TANH)
.dist(new NormalDistribution(0, 0.5)).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
.activation(Activation.TANH)
.dist(
new NormalDistribution(0,
0.5))
.build())
.layer(2, new RnnOutputLayer.Builder(LossFunction.MCXENT)
.nIn(8).nOut(nOut).activation(Activation.SOFTMAX)
.dist(new NormalDistribution(0, 0.5))
.build())
.backpropType(BackpropType.TruncatedBPTT)
.tBPTTBackwardLength(timeSeriesLength).tBPTTForwardLength(timeSeriesLength).build();
Nd4j.getRandom().setSeed(12345);
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
INDArray inputLong = Nd4j.rand(new int[] {miniBatchSize, nIn, nTimeSlices * timeSeriesLength});
INDArray labelsLong = Nd4j.rand(new int[] {miniBatchSize, nOut, nTimeSlices * timeSeriesLength});
mln.fit(inputLong, labelsLong);
}
Example #11
| Source File: GradientCheckTests.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testEmbeddingLayerSimple() {
Random r = new Random(12345);
int nExamples = 5;
INDArray input = Nd4j.zeros(nExamples, 1);
INDArray labels = Nd4j.zeros(nExamples, 3);
for (int i = 0; i < nExamples; i++) {
input.putScalar(i, r.nextInt(4));
labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
.list().layer(0,
new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
.updater(new NoOp()).activation(
Activation.TANH)
.build())
.layer(1, new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
.weightInit(WeightInit.XAVIER)
.updater(new NoOp()).activation(Activation.SOFTMAX).build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
if (PRINT_RESULTS) {
System.out.println("testEmbeddingLayerSimple");
// for (int j = 0; j < mln.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
String msg = "testEmbeddingLayerSimple";
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(mln);
}
Example #12
| Source File: GradientCheckTests.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testEmbeddingLayerPreluSimple() {
Random r = new Random(12345);
int nExamples = 5;
INDArray input = Nd4j.zeros(nExamples, 1);
INDArray labels = Nd4j.zeros(nExamples, 3);
for (int i = 0; i < nExamples; i++) {
input.putScalar(i, r.nextInt(4));
labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
.list().layer(new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
.updater(new NoOp()).build())
.layer(new PReLULayer.Builder().inputShape(3).sharedAxes(1).updater(new NoOp()).build())
.layer(new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
.weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1))
.updater(new NoOp()).activation(Activation.SOFTMAX).build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
if (PRINT_RESULTS) {
System.out.println("testEmbeddingLayerSimple");
// for (int j = 0; j < mln.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
String msg = "testEmbeddingLayerSimple";
assertTrue(msg, gradOK);
}
Example #13
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testTBPTTLongerThanTS() {
//Extremely simple test of the 'does it throw an exception' variety
int timeSeriesLength = 20;
int tbpttLength = 1000;
int miniBatchSize = 7;
int nIn = 5;
int nOut = 4;
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.weightInit(WeightInit.XAVIER).list()
.layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
.activation(Activation.TANH).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
.activation(Activation.TANH).build())
.layer(2, new RnnOutputLayer.Builder(LossFunction.MSE).nIn(8).nOut(nOut)
.activation(Activation.IDENTITY).build())
.backpropType(BackpropType.TruncatedBPTT)
.tBPTTBackwardLength(tbpttLength).tBPTTForwardLength(tbpttLength).build();
Nd4j.getRandom().setSeed(12345);
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
INDArray features = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength});
INDArray labels = Nd4j.rand(new int[] {miniBatchSize, nOut, timeSeriesLength});
INDArray maskArrayInput = Nd4j.ones(miniBatchSize, timeSeriesLength);
INDArray maskArrayOutput = Nd4j.ones(miniBatchSize, timeSeriesLength);
DataSet ds = new DataSet(features, labels, maskArrayInput, maskArrayOutput);
INDArray initialParams = mln.params().dup();
mln.fit(ds);
INDArray afterParams = mln.params();
assertNotEquals(initialParams, afterParams);
}
Example #14
| Source File: MultiNeuralNetConfLayerBuilderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testNeuralNetConfigAPI() {
LossFunction newLoss = LossFunction.SQUARED_LOSS;
int newNumIn = numIn + 1;
int newNumOut = numOut + 1;
WeightInit newWeight = WeightInit.UNIFORM;
double newDrop = 0.5;
int[] newFS = new int[] {3, 3};
int newFD = 7;
int[] newStride = new int[] {3, 3};
Convolution.Type newConvType = Convolution.Type.SAME;
PoolingType newPoolType = PoolingType.AVG;
double newCorrupt = 0.5;
double newSparsity = 0.5;
MultiLayerConfiguration multiConf1 =
new NeuralNetConfiguration.Builder().list()
.layer(0, new DenseLayer.Builder().nIn(newNumIn).nOut(newNumOut).activation(act)
.build())
.layer(1, new DenseLayer.Builder().nIn(newNumIn + 1).nOut(newNumOut + 1)
.activation(act).build())
.build();
NeuralNetConfiguration firstLayer = multiConf1.getConf(0);
NeuralNetConfiguration secondLayer = multiConf1.getConf(1);
assertFalse(firstLayer.equals(secondLayer));
}
Example #15
| Source File: TestOptimizers.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
private static MultiLayerConfiguration getMLPConfigIris(OptimizationAlgorithm oa) {
MultiLayerConfiguration c = new NeuralNetConfiguration.Builder().optimizationAlgo(oa)
.updater(new AdaGrad(1e-1)).seed(12345L)
.list().layer(0,
new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
.activation(Activation.RELU)
.build())
.layer(1, new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
.weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build())
.build();
return c;
}
Example #16
| Source File: TestMultiLayerSpace.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testBasic() {
MultiLayerConfiguration expected =
new NeuralNetConfiguration.Builder()
.updater(new Sgd(0.005)).seed(12345).list()
.layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build())
.layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2,
new OutputLayer.Builder().lossFunction(LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(10).nOut(5).build())
.build();
MultiLayerSpace mls =
new MultiLayerSpace.Builder()
.updater(new Sgd(0.005)).seed(12345)
.addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(10).build(),
new FixedValue<>(2)) //2 identical layers
.addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT)
.activation(Activation.SOFTMAX)
.nIn(10).nOut(5).build()).build();
int nParams = mls.numParameters();
assertEquals(0, nParams);
MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration();
assertEquals(expected, conf);
}
Example #17
| Source File: TestLossFunctionsSizeChecks.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testL2() {
LossFunction[] lossFunctionList = {LossFunction.MSE, LossFunction.L1, LossFunction.XENT,
LossFunction.MCXENT, LossFunction.SQUARED_LOSS, LossFunction.RECONSTRUCTION_CROSSENTROPY,
LossFunction.NEGATIVELOGLIKELIHOOD, LossFunction.COSINE_PROXIMITY, LossFunction.HINGE,
LossFunction.SQUARED_HINGE, LossFunction.KL_DIVERGENCE, LossFunction.MEAN_ABSOLUTE_ERROR,
LossFunction.L2, LossFunction.MEAN_ABSOLUTE_PERCENTAGE_ERROR,
LossFunction.MEAN_SQUARED_LOGARITHMIC_ERROR, LossFunction.POISSON};
testLossFunctions(lossFunctionList);
}
Example #18
| Source File: TestLossFunctionsSizeChecks.java From nd4j with Apache License 2.0 | 5 votes |
|
@Test
public void testL2() {
LossFunction[] lossFunctionList = {LossFunction.MSE, LossFunction.L1, LossFunction.EXPLL, LossFunction.XENT,
LossFunction.MCXENT, LossFunction.SQUARED_LOSS, LossFunction.RECONSTRUCTION_CROSSENTROPY,
LossFunction.NEGATIVELOGLIKELIHOOD, LossFunction.COSINE_PROXIMITY, LossFunction.HINGE,
LossFunction.SQUARED_HINGE, LossFunction.KL_DIVERGENCE, LossFunction.MEAN_ABSOLUTE_ERROR,
LossFunction.L2, LossFunction.MEAN_ABSOLUTE_PERCENTAGE_ERROR,
LossFunction.MEAN_SQUARED_LOGARITHMIC_ERROR, LossFunction.POISSON};
testLossFunctions(lossFunctionList);
}
Example #19
| Source File: TestSetGetParameters.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testSetParameters() {
//Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this.
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
.layer(0, new DenseLayer.Builder().nIn(9).nOut(10)
.dist(new NormalDistribution(0, 1)).build())
.layer(1, new DenseLayer.Builder().nIn(10).nOut(11)
.dist(new NormalDistribution(0, 1)).build())
.layer(2, new AutoEncoder.Builder().corruptionLevel(0.5).nIn(11).nOut(12)
.dist(new NormalDistribution(0, 1)).build())
.layer(3, new OutputLayer.Builder(LossFunction.MSE).nIn(12).nOut(12)
.dist(new NormalDistribution(0, 1)).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray initParams = net.params().dup();
Map<String, INDArray> initParams2 = net.paramTable();
net.setParams(net.params());
INDArray initParamsAfter = net.params();
Map<String, INDArray> initParams2After = net.paramTable();
for (String s : initParams2.keySet()) {
assertTrue("Params differ: " + s, initParams2.get(s).equals(initParams2After.get(s)));
}
assertEquals(initParams, initParamsAfter);
//Now, try the other way: get(set(random))
INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape());
net.setParams(randomParams.dup());
assertEquals(net.params(), randomParams);
}
Example #20
| Source File: CenterLossOutputLayerTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
public ComputationGraph getCNNMnistConfig() {
int nChannels = 1; // Number of input channels
int outputNum = 10; // The number of possible outcomes
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) // Training iterations as above
.l2(0.0005).weightInit(WeightInit.XAVIER)
.updater(new Nesterovs(0.01, 0.9))
.graphBuilder().addInputs("input")
.setInputTypes(InputType.convolutionalFlat(28, 28, 1))
.addLayer("0", new ConvolutionLayer.Builder(5, 5)
//nIn and nOut specify channels. nIn here is the nChannels and nOut is the number of filters to be applied
.nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(),
"input")
.addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
.stride(2, 2).build(), "0")
.addLayer("2", new ConvolutionLayer.Builder(5, 5)
//Note that nIn need not be specified in later layers
.stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1")
.addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
.stride(2, 2).build(), "2")
.addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3")
.addLayer("output",
new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder(
LossFunction.MCXENT).nOut(outputNum)
.activation(Activation.SOFTMAX).build(),
"4")
.setOutputs("output").build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
return graph;
}
Example #21
| Source File: TestComputationGraphSpace.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testBasic() {
ComputationGraphConfiguration expected = new NeuralNetConfiguration.Builder()
.updater(new Sgd(0.005))
.seed(12345)
.graphBuilder().addInputs("in")
.addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in")
.addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).build(), "0").addLayer("2",
new OutputLayer.Builder().lossFunction(LossFunction.MCXENT)
.activation(Activation.SOFTMAX)
.nIn(10).nOut(5)
.build(),
"1")
.setOutputs("2").build();
ComputationGraphSpace cgs = new ComputationGraphSpace.Builder()
.updater(new Sgd(0.005))
.seed(12345).addInputs("in")
.addLayer("0", new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), "in")
.addLayer("1", new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), "0")
.addLayer("2", new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(5)
.build(), "1")
.setOutputs("2").setInputTypes(InputType.feedForward(10))
.build();
int nParams = cgs.numParameters();
assertEquals(0, nParams);
ComputationGraphConfiguration conf = cgs.getValue(new double[0]).getConfiguration();
assertEquals(expected, conf);
}
Example #22
| Source File: OutputLayerTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testCnnOutputLayerSoftmax(){
//Check that softmax is applied channels-wise
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder().seed(12345L)
.updater(new NoOp())
.convolutionMode(ConvolutionMode.Same)
.list()
.layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY)
.dist(new NormalDistribution(0, 1.0))
.updater(new NoOp()).build())
.layer(new CnnLossLayer.Builder(LossFunction.MSE)
.activation(Activation.SOFTMAX)
.build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray in = Nd4j.rand(new int[]{2,3,4,5});
INDArray out = net.output(in);
double min = out.minNumber().doubleValue();
double max = out.maxNumber().doubleValue();
assertTrue(min >= 0 && max <= 1.0);
INDArray sum = out.sum(1);
assertEquals(Nd4j.ones(DataType.FLOAT,2,4,5), sum);
}
Example #23
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testRnnTimeStep2dInput() {
Nd4j.getRandom().setSeed(12345);
int timeSeriesLength = 6;
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder()
.list().layer(0,
new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder()
.nIn(5).nOut(7).activation(Activation.TANH)
.dist(new NormalDistribution(0, 0.5)).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7)
.nOut(8).activation(Activation.TANH)
.dist(new NormalDistribution(0,
0.5))
.build())
.layer(2, new RnnOutputLayer.Builder(LossFunction.MCXENT)
.nIn(8).nOut(4)
.activation(Activation.SOFTMAX)
.dist(new NormalDistribution(0, 0.5)).build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
INDArray input3d = Nd4j.rand(new long[] {3, 5, timeSeriesLength});
INDArray out3d = mln.rnnTimeStep(input3d);
assertArrayEquals(out3d.shape(), new long[] {3, 4, timeSeriesLength});
mln.rnnClearPreviousState();
for (int i = 0; i < timeSeriesLength; i++) {
INDArray input2d = input3d.tensorAlongDimension(i, 1, 0);
INDArray out2d = mln.rnnTimeStep(input2d);
assertArrayEquals(out2d.shape(), new long[] {3, 4});
INDArray expOut2d = out3d.tensorAlongDimension(i, 1, 0);
assertEquals(out2d, expOut2d);
}
//Check same but for input of size [3,5,1]. Expect [3,4,1] out
mln.rnnClearPreviousState();
for (int i = 0; i < timeSeriesLength; i++) {
INDArray temp = Nd4j.create(new int[] {3, 5, 1});
temp.tensorAlongDimension(0, 1, 0).assign(input3d.tensorAlongDimension(i, 1, 0));
INDArray out3dSlice = mln.rnnTimeStep(temp);
assertArrayEquals(out3dSlice.shape(), new long[] {3, 4, 1});
assertTrue(out3dSlice.tensorAlongDimension(0, 1, 0).equals(out3d.tensorAlongDimension(i, 1, 0)));
}
}
Example #24
| Source File: TestSetGetParameters.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testInitWithParams() {
Nd4j.getRandom().setSeed(12345);
//Create configuration. Doesn't matter if this doesn't actually work for forward/backward pass here
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list()
.layer(0, new ConvolutionLayer.Builder().nIn(10).nOut(10).kernelSize(2, 2).stride(2, 2)
.padding(2, 2).build())
.layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build())
.layer(2, new GravesLSTM.Builder().nIn(10).nOut(10).build())
.layer(3, new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build())
.layer(4, new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray params = net.params();
MultiLayerNetwork net2 = new MultiLayerNetwork(conf);
net2.init(params, true);
MultiLayerNetwork net3 = new MultiLayerNetwork(conf);
net3.init(params, false);
assertEquals(params, net2.params());
assertEquals(params, net3.params());
assertFalse(params == net2.params()); //Different objects due to clone
assertTrue(params == net3.params()); //Same object due to clone
Map<String, INDArray> paramsMap = net.paramTable();
Map<String, INDArray> paramsMap2 = net2.paramTable();
Map<String, INDArray> paramsMap3 = net3.paramTable();
for (String s : paramsMap.keySet()) {
assertEquals(paramsMap.get(s), paramsMap2.get(s));
assertEquals(paramsMap.get(s), paramsMap3.get(s));
}
}
Example #25
| Source File: LearnDigitsDropout.java From aifh with Apache License 2.0 | 4 votes |
|
/**
* The main method.
* @param args Not used.
*/
public static void main(String[] args) {
try {
int seed = 43;
double learningRate = 1e-2;
int nEpochs = 50;
int batchSize = 500;
// Setup training data.
System.out.println("Please wait, reading MNIST training data.");
String dir = System.getProperty("user.dir");
MNISTReader trainingReader = MNIST.loadMNIST(dir, true);
MNISTReader validationReader = MNIST.loadMNIST(dir, false);
DataSet trainingSet = trainingReader.getData();
DataSet validationSet = validationReader.getData();
DataSetIterator trainSetIterator = new ListDataSetIterator(trainingSet.asList(), batchSize);
DataSetIterator validationSetIterator = new ListDataSetIterator(validationSet.asList(), validationReader.getNumRows());
System.out.println("Training set size: " + trainingReader.getNumImages());
System.out.println("Validation set size: " + validationReader.getNumImages());
System.out.println(trainingSet.get(0).getFeatures().size(1));
System.out.println(validationSet.get(0).getFeatures().size(1));
int numInputs = trainingReader.getNumCols()*trainingReader.getNumRows();
int numOutputs = 10;
int numHiddenNodes = 100;
// Create neural network.
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.seed(seed)
.iterations(1)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.learningRate(learningRate)
.updater(Updater.NESTEROVS).momentum(0.9)
.list(2)
.layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes)
.weightInit(WeightInit.XAVIER)
.activation("relu")
.build())
.layer(1, new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD)
.weightInit(WeightInit.XAVIER)
.activation("softmax")
.nIn(numHiddenNodes).nOut(numOutputs).build())
.pretrain(false).backprop(true).build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
model.setListeners(new ScoreIterationListener(1));
// Define when we want to stop training.
EarlyStoppingModelSaver saver = new InMemoryModelSaver();
EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder()
//.epochTerminationConditions(new MaxEpochsTerminationCondition(10))
.epochTerminationConditions(new ScoreImprovementEpochTerminationCondition(5))
.evaluateEveryNEpochs(1)
.scoreCalculator(new DataSetLossCalculator(validationSetIterator, true)) //Calculate test set score
.modelSaver(saver)
.build();
EarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, conf, trainSetIterator);
// Train and display result.
EarlyStoppingResult result = trainer.fit();
System.out.println("Termination reason: " + result.getTerminationReason());
System.out.println("Termination details: " + result.getTerminationDetails());
System.out.println("Total epochs: " + result.getTotalEpochs());
System.out.println("Best epoch number: " + result.getBestModelEpoch());
System.out.println("Score at best epoch: " + result.getBestModelScore());
model = saver.getBestModel();
// Evaluate
Evaluation eval = new Evaluation(numOutputs);
validationSetIterator.reset();
for (int i = 0; i < validationSet.numExamples(); i++) {
DataSet t = validationSet.get(i);
INDArray features = t.getFeatureMatrix();
INDArray labels = t.getLabels();
INDArray predicted = model.output(features, false);
eval.eval(labels, predicted);
}
//Print the evaluation statistics
System.out.println(eval.stats());
} catch(Exception ex) {
ex.printStackTrace();
}
}
Example #26
| Source File: TestMultiLayerSpace.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testGridCandidateGenerator(){
ParameterSpace<Integer> layerSizeParam = new DiscreteParameterSpace<>(32, 48, 64);
ParameterSpace<Double> learningRateParam = new DiscreteParameterSpace<>(0.005, 0.007, 0.01);
MultiLayerSpace hyperParamaterSpace = new MultiLayerSpace.Builder()
.seed(12345)
.biasInit(1)
.l2(1e-4)
.updater(new NesterovsSpace(learningRateParam))
.addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(layerSizeParam)
.weightInit(WeightInit.XAVIER)
.activation(Activation.RELU)
.build())
.addLayer(new DenseLayerSpace.Builder().nIn(layerSizeParam).nOut(layerSizeParam)
.weightInit(WeightInit.XAVIER)
.activation(Activation.RELU)
.build())
.addLayer(new OutputLayerSpace.Builder()
.lossFunction(LossFunctions.LossFunction.MSE)
.weightInit(WeightInit.XAVIER)
.activation(Activation.SOFTMAX)
.nIn(layerSizeParam).nOut(10).build())
.build();
CandidateGenerator candidateGenerator = new GridSearchCandidateGenerator(hyperParamaterSpace, 30, GridSearchCandidateGenerator.Mode.Sequential, null);
// CandidateGenerator candidateGenerator = new RandomSearchGenerator(hyperParamaterSpace);
Set<Pair<Double,Integer>> expCandidates = new HashSet<>();
for(Double d : new double[]{0.005, 0.007, 0.01}){
for(int i : new int[]{32, 48, 64}){
expCandidates.add(new Pair<>(d, i));
}
}
Set<Pair<Double,Integer>> actCandidates = new HashSet<>();
while(candidateGenerator.hasMoreCandidates()) {
Candidate<DL4JConfiguration> conf = candidateGenerator.getCandidate();
MultiLayerConfiguration mlc = conf.getValue().getMultiLayerConfiguration();
FeedForwardLayer ffl = ((FeedForwardLayer) mlc.getConf(0).getLayer());
// System.out.println(ffl.getIUpdater() + ", " + ffl.getNOut());
actCandidates.add(new Pair<>(ffl.getIUpdater().getLearningRate(0,0), (int)ffl.getNOut()));
}
assertEquals(expCandidates, actCandidates);
}
Example #27
| Source File: MultiLayerTestRNN.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testRnnStateMethods() {
Nd4j.getRandom().setSeed(12345);
int timeSeriesLength = 6;
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder()
.list().layer(0,
new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder()
.nIn(5).nOut(7).activation(Activation.TANH)
.dist(new NormalDistribution(0, 0.5)).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7)
.nOut(8).activation(Activation.TANH)
.dist(new NormalDistribution(0,
0.5))
.build())
.layer(2, new RnnOutputLayer.Builder(LossFunction.MCXENT)
.nIn(8).nOut(4)
.activation(Activation.SOFTMAX)
.dist(new NormalDistribution(0, 0.5)).build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
INDArray input = Nd4j.rand(new int[] {3, 5, timeSeriesLength});
List<INDArray> allOutputActivations = mln.feedForward(input, true);
INDArray outAct = allOutputActivations.get(3);
INDArray outRnnTimeStep = mln.rnnTimeStep(input);
assertTrue(outAct.equals(outRnnTimeStep)); //Should be identical here
Map<String, INDArray> currStateL0 = mln.rnnGetPreviousState(0);
Map<String, INDArray> currStateL1 = mln.rnnGetPreviousState(1);
assertTrue(currStateL0.size() == 2);
assertTrue(currStateL1.size() == 2);
INDArray lastActL0 = currStateL0.get(GravesLSTM.STATE_KEY_PREV_ACTIVATION);
INDArray lastMemL0 = currStateL0.get(GravesLSTM.STATE_KEY_PREV_MEMCELL);
assertTrue(lastActL0 != null && lastMemL0 != null);
INDArray lastActL1 = currStateL1.get(GravesLSTM.STATE_KEY_PREV_ACTIVATION);
INDArray lastMemL1 = currStateL1.get(GravesLSTM.STATE_KEY_PREV_MEMCELL);
assertTrue(lastActL1 != null && lastMemL1 != null);
INDArray expectedLastActL0 = allOutputActivations.get(1).tensorAlongDimension(timeSeriesLength - 1, 1, 0);
assertTrue(expectedLastActL0.equals(lastActL0));
INDArray expectedLastActL1 = allOutputActivations.get(2).tensorAlongDimension(timeSeriesLength - 1, 1, 0);
assertTrue(expectedLastActL1.equals(lastActL1));
//Check clearing and setting of state:
mln.rnnClearPreviousState();
assertTrue(mln.rnnGetPreviousState(0).isEmpty());
assertTrue(mln.rnnGetPreviousState(1).isEmpty());
mln.rnnSetPreviousState(0, currStateL0);
assertTrue(mln.rnnGetPreviousState(0).size() == 2);
mln.rnnSetPreviousState(1, currStateL1);
assertTrue(mln.rnnGetPreviousState(1).size() == 2);
}
Example #28
| Source File: OutputLayerTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnnLossLayer(){
for(WorkspaceMode ws : WorkspaceMode.values()) {
log.info("*** Testing workspace: " + ws);
for (Activation a : new Activation[]{Activation.TANH, Activation.SELU}) {
//Check that (A+identity) is equal to (identity+A), for activation A
//i.e., should get same output and weight gradients for both
MultiLayerConfiguration conf1 =
new NeuralNetConfiguration.Builder().seed(12345L)
.updater(new NoOp())
.convolutionMode(ConvolutionMode.Same)
.inferenceWorkspaceMode(ws)
.trainingWorkspaceMode(ws)
.list()
.layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY)
.kernelSize(2, 2).stride(1, 1)
.dist(new NormalDistribution(0, 1.0))
.updater(new NoOp()).build())
.layer(new CnnLossLayer.Builder(LossFunction.MSE)
.activation(a)
.build())
.build();
MultiLayerConfiguration conf2 =
new NeuralNetConfiguration.Builder().seed(12345L)
.updater(new NoOp())
.convolutionMode(ConvolutionMode.Same)
.inferenceWorkspaceMode(ws)
.trainingWorkspaceMode(ws)
.list()
.layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(a)
.kernelSize(2, 2).stride(1, 1)
.dist(new NormalDistribution(0, 1.0))
.updater(new NoOp()).build())
.layer(new CnnLossLayer.Builder(LossFunction.MSE)
.activation(Activation.IDENTITY)
.build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf1);
mln.init();
MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2);
mln2.init();
mln2.setParams(mln.params());
INDArray in = Nd4j.rand(new int[]{3, 3, 5, 5});
INDArray out1 = mln.output(in);
INDArray out2 = mln2.output(in);
assertEquals(out1, out2);
INDArray labels = Nd4j.rand(out1.shape());
mln.setInput(in);
mln.setLabels(labels);
mln2.setInput(in);
mln2.setLabels(labels);
mln.computeGradientAndScore();
mln2.computeGradientAndScore();
assertEquals(mln.score(), mln2.score(), 1e-6);
assertEquals(mln.gradient().gradient(), mln2.gradient().gradient());
//Also check computeScoreForExamples
INDArray in2a = Nd4j.rand(new int[]{1, 3, 5, 5});
INDArray labels2a = Nd4j.rand(new int[]{1, 4, 5, 5});
INDArray in2 = Nd4j.concat(0, in2a, in2a);
INDArray labels2 = Nd4j.concat(0, labels2a, labels2a);
INDArray s = mln.scoreExamples(new DataSet(in2, labels2), false);
assertArrayEquals(new long[]{2, 1}, s.shape());
assertEquals(s.getDouble(0), s.getDouble(1), 1e-6);
TestUtils.testModelSerialization(mln);
}
}
}
Example #29
| Source File: OutputLayerTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnnLossLayerCompGraph(){
for(WorkspaceMode ws : WorkspaceMode.values()) {
log.info("*** Testing workspace: " + ws);
for (Activation a : new Activation[]{Activation.TANH, Activation.SELU}) {
//Check that (A+identity) is equal to (identity+A), for activation A
//i.e., should get same output and weight gradients for both
ComputationGraphConfiguration conf1 =
new NeuralNetConfiguration.Builder().seed(12345L)
.updater(new NoOp())
.convolutionMode(ConvolutionMode.Same)
.inferenceWorkspaceMode(ws)
.trainingWorkspaceMode(ws)
.graphBuilder()
.addInputs("in")
.addLayer("0", new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY)
.kernelSize(2, 2).stride(1, 1)
.dist(new NormalDistribution(0, 1.0))
.updater(new NoOp()).build(), "in")
.addLayer("1", new CnnLossLayer.Builder(LossFunction.MSE)
.activation(a)
.build(), "0")
.setOutputs("1")
.build();
ComputationGraphConfiguration conf2 =
new NeuralNetConfiguration.Builder().seed(12345L)
.updater(new NoOp())
.convolutionMode(ConvolutionMode.Same)
.inferenceWorkspaceMode(ws)
.trainingWorkspaceMode(ws)
.graphBuilder()
.addInputs("in")
.addLayer("0", new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(a)
.kernelSize(2, 2).stride(1, 1)
.dist(new NormalDistribution(0, 1.0))
.updater(new NoOp()).build(), "in")
.addLayer("1", new CnnLossLayer.Builder(LossFunction.MSE)
.activation(Activation.IDENTITY)
.build(), "0")
.setOutputs("1")
.build();
ComputationGraph graph = new ComputationGraph(conf1);
graph.init();
ComputationGraph graph2 = new ComputationGraph(conf2);
graph2.init();
graph2.setParams(graph.params());
INDArray in = Nd4j.rand(new int[]{3, 3, 5, 5});
INDArray out1 = graph.outputSingle(in);
INDArray out2 = graph2.outputSingle(in);
assertEquals(out1, out2);
INDArray labels = Nd4j.rand(out1.shape());
graph.setInput(0,in);
graph.setLabels(labels);
graph2.setInput(0,in);
graph2.setLabels(labels);
graph.computeGradientAndScore();
graph2.computeGradientAndScore();
assertEquals(graph.score(), graph2.score(), 1e-6);
assertEquals(graph.gradient().gradient(), graph2.gradient().gradient());
//Also check computeScoreForExamples
INDArray in2a = Nd4j.rand(new int[]{1, 3, 5, 5});
INDArray labels2a = Nd4j.rand(new int[]{1, 4, 5, 5});
INDArray in2 = Nd4j.concat(0, in2a, in2a);
INDArray labels2 = Nd4j.concat(0, labels2a, labels2a);
INDArray s = graph.scoreExamples(new DataSet(in2, labels2), false);
assertArrayEquals(new long[]{2, 1}, s.shape());
assertEquals(s.getDouble(0), s.getDouble(1), 1e-6);
TestUtils.testModelSerialization(graph);
}
}
}
Example #30
| Source File: VaeGradientCheckTests.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testVaeAsMLP() {
//Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output
//This gradient check tests this part
Activation[] activFns = {Activation.IDENTITY, Activation.TANH, Activation.IDENTITY, Activation.TANH, Activation.IDENTITY, Activation.TANH};
LossFunction[] lossFunctions = {LossFunction.MCXENT, LossFunction.MCXENT, LossFunction.MSE, LossFunction.MSE, LossFunction.MCXENT, LossFunction.MSE};
Activation[] outputActivations = {Activation.SOFTMAX, Activation.SOFTMAX, Activation.TANH, Activation.TANH, Activation.SOFTMAX, Activation.TANH};
//use l2vals[i] with l1vals[i]
double[] l2vals = {0.4, 0.0, 0.4, 0.4, 0.0, 0.0};
double[] l1vals = {0.0, 0.0, 0.5, 0.0, 0.0, 0.5};
double[] biasL2 = {0.0, 0.0, 0.0, 0.2, 0.0, 0.4};
double[] biasL1 = {0.0, 0.0, 0.6, 0.0, 0.0, 0.0};
int[][] encoderLayerSizes = new int[][] {{5}, {5}, {5, 6}, {5, 6}, {5}, {5, 6}};
int[][] decoderLayerSizes = new int[][] {{6}, {7, 8}, {6}, {7, 8}, {6}, {7, 8}};
int[] minibatches = new int[]{1,5,4,3,1,4};
Nd4j.getRandom().setSeed(12345);
for( int i=0; i<activFns.length; i++ ){
LossFunction lf = lossFunctions[i];
Activation outputActivation = outputActivations[i];
double l2 = l2vals[i];
double l1 = l1vals[i];
int[] encoderSizes = encoderLayerSizes[i];
int[] decoderSizes = decoderLayerSizes[i];
int minibatch = minibatches[i];
INDArray input = Nd4j.rand(minibatch, 4);
INDArray labels = Nd4j.create(minibatch, 3);
for (int j = 0; j < minibatch; j++) {
labels.putScalar(j, j % 3, 1.0);
}
Activation afn = activFns[i];
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder().l2(l2).l1(l1)
.dataType(DataType.DOUBLE)
.updater(new NoOp())
.l2Bias(biasL2[i]).l1Bias(biasL1[i])
.updater(new NoOp()).seed(12345L).list()
.layer(0, new VariationalAutoencoder.Builder().nIn(4)
.nOut(3).encoderLayerSizes(encoderSizes)
.decoderLayerSizes(decoderSizes)
.dist(new NormalDistribution(0, 1))
.activation(afn)
.build())
.layer(1, new OutputLayer.Builder(lf)
.activation(outputActivation).nIn(3).nOut(3)
.dist(new NormalDistribution(0, 1))
.build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
String msg = "testVaeAsMLP() - activationFn=" + afn + ", lossFn=" + lf
+ ", outputActivation=" + outputActivation + ", encLayerSizes = "
+ Arrays.toString(encoderSizes) + ", decLayerSizes = "
+ Arrays.toString(decoderSizes) + ", l2=" + l2 + ", l1=" + l1;
if (PRINT_RESULTS) {
System.out.println(msg);
// for (int j = 0; j < mln.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input,
labels);
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(mln);
}
}
