org.deeplearning4j.nn.conf.distribution.NormalDistribution Java Examples
The following examples show how to use
org.deeplearning4j.nn.conf.distribution.NormalDistribution.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: ComputationGraphConfigurationTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Test
public void testJSONBasic() {
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1)).updater(new NoOp())
.graphBuilder().addInputs("input")
.appendLayer("firstLayer",
new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build())
.addLayer("outputLayer",
new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(5).nOut(3).build(),
"firstLayer")
.setOutputs("outputLayer").build();
String json = conf.toJson();
ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json);
assertEquals(json, conf2.toJson());
assertEquals(conf, conf2);
}
Example #2
| Source File: MultiLayerTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
private static MultiLayerConfiguration getConf() {
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder().seed(12345L)
.list().layer(0,
new DenseLayer.Builder().nIn(4).nOut(3)
.dist(new NormalDistribution(0,1))
.build())
.layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(3).nOut(3)
.dist(new NormalDistribution(0, 1)).build())
.build();
return conf;
}
Example #3
| Source File: LRNGradientCheckTests.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testGradientLRNSimple() {
Nd4j.getRandom().setSeed(12345);
int minibatch = 10;
int depth = 6;
int hw = 5;
int nOut = 4;
INDArray input = Nd4j.rand(new int[] {minibatch, depth, hw, hw});
INDArray labels = Nd4j.zeros(minibatch, nOut);
Random r = new Random(12345);
for (int i = 0; i < minibatch; i++) {
labels.putScalar(i, r.nextInt(nOut), 1.0);
}
MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
.dataType(DataType.DOUBLE)
.seed(12345L)
.dist(new NormalDistribution(0, 2)).list()
.layer(0, new ConvolutionLayer.Builder().nOut(6).kernelSize(2, 2).stride(1, 1)
.activation(Activation.TANH).build())
.layer(1, new LocalResponseNormalization.Builder().build())
.layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(nOut).build())
.setInputType(InputType.convolutional(hw, hw, depth));
MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
mln.init();
// if (PRINT_RESULTS) {
// for (int j = 0; j < mln.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
// }
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(gradOK);
TestUtils.testModelSerialization(mln);
}
Example #4
| Source File: TestSetGetParameters.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testSetParametersRNN() {
//Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this.
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
.layer(0, new GravesLSTM.Builder().nIn(9).nOut(10)
.dist(new NormalDistribution(0, 1)).build())
.layer(1, new GravesLSTM.Builder().nIn(10).nOut(11)
.dist(new NormalDistribution(0, 1)).build())
.layer(2, new RnnOutputLayer.Builder(LossFunction.MSE)
.dist(new NormalDistribution(0, 1)).nIn(11).nOut(12).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray initParams = net.params().dup();
Map<String, INDArray> initParams2 = net.paramTable();
net.setParams(net.params());
INDArray initParamsAfter = net.params();
Map<String, INDArray> initParams2After = net.paramTable();
for (String s : initParams2.keySet()) {
assertTrue("Params differ: " + s, initParams2.get(s).equals(initParams2After.get(s)));
}
assertEquals(initParams, initParamsAfter);
//Now, try the other way: get(set(random))
INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape());
net.setParams(randomParams.dup());
assertEquals(net.params(), randomParams);
}
Example #5
| Source File: GradientCheckTests.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testEmbeddingLayerPreluSimple() {
Random r = new Random(12345);
int nExamples = 5;
INDArray input = Nd4j.zeros(nExamples, 1);
INDArray labels = Nd4j.zeros(nExamples, 3);
for (int i = 0; i < nExamples; i++) {
input.putScalar(i, r.nextInt(4));
labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
.list().layer(new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
.updater(new NoOp()).build())
.layer(new PReLULayer.Builder().inputShape(3).sharedAxes(1).updater(new NoOp()).build())
.layer(new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
.weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1))
.updater(new NoOp()).activation(Activation.SOFTMAX).build())
.build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
if (PRINT_RESULTS) {
System.out.println("testEmbeddingLayerSimple");
// for (int j = 0; j < mln.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
String msg = "testEmbeddingLayerSimple";
assertTrue(msg, gradOK);
}
Example #6
| Source File: TestSetGetParameters.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testSetParameters() {
//Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this.
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
.layer(0, new DenseLayer.Builder().nIn(9).nOut(10)
.dist(new NormalDistribution(0, 1)).build())
.layer(1, new DenseLayer.Builder().nIn(10).nOut(11)
.dist(new NormalDistribution(0, 1)).build())
.layer(2, new AutoEncoder.Builder().corruptionLevel(0.5).nIn(11).nOut(12)
.dist(new NormalDistribution(0, 1)).build())
.layer(3, new OutputLayer.Builder(LossFunction.MSE).nIn(12).nOut(12)
.dist(new NormalDistribution(0, 1)).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray initParams = net.params().dup();
Map<String, INDArray> initParams2 = net.paramTable();
net.setParams(net.params());
INDArray initParamsAfter = net.params();
Map<String, INDArray> initParams2After = net.paramTable();
for (String s : initParams2.keySet()) {
assertTrue("Params differ: " + s, initParams2.get(s).equals(initParams2After.get(s)));
}
assertEquals(initParams, initParamsAfter);
//Now, try the other way: get(set(random))
INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape());
net.setParams(randomParams.dup());
assertEquals(net.params(), randomParams);
}
Example #7
| Source File: ComputationGraphTestRNN.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testTBPTTLongerThanTS() {
int tbpttLength = 100;
int timeSeriesLength = 20;
int miniBatchSize = 7;
int nIn = 5;
int nOut = 4;
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder()
.addInputs("in")
.addLayer("0", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
.activation(Activation.TANH)
.dist(new NormalDistribution(0, 0.5)).build(), "in")
.addLayer("1", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
.activation(Activation.TANH)
.dist(new NormalDistribution(0,
0.5))
.build(), "0")
.addLayer("out", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.nIn(8).nOut(nOut)
.activation(Activation.SOFTMAX)
.dist(new NormalDistribution(0, 0.5)).build(), "1")
.setOutputs("out").backpropType(BackpropType.TruncatedBPTT)
.tBPTTBackwardLength(tbpttLength).tBPTTForwardLength(tbpttLength).build();
Nd4j.getRandom().setSeed(12345);
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
INDArray inputLong = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength});
INDArray labelsLong = Nd4j.rand(new int[] {miniBatchSize, nOut, timeSeriesLength});
INDArray initialParams = graph.params().dup();
graph.fit(new INDArray[] {inputLong}, new INDArray[] {labelsLong});
INDArray afterParams = graph.params();
assertNotEquals(initialParams, afterParams);
}
Example #8
| Source File: MultiLayerNeuralNetConfigurationTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testJson() throws Exception {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
.layer(0, new DenseLayer.Builder().dist(new NormalDistribution(1, 1e-1)).build())
.inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build();
String json = conf.toJson();
MultiLayerConfiguration from = MultiLayerConfiguration.fromJson(json);
assertEquals(conf.getConf(0), from.getConf(0));
Properties props = new Properties();
props.put("json", json);
String key = props.getProperty("json");
assertEquals(json, key);
File f = testDir.newFile("props");
f.deleteOnExit();
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f));
props.store(bos, "");
bos.flush();
bos.close();
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f));
Properties props2 = new Properties();
props2.load(bis);
bis.close();
assertEquals(props2.getProperty("json"), props.getProperty("json"));
String json2 = props2.getProperty("json");
MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromJson(json2);
assertEquals(conf.getConf(0), conf3.getConf(0));
}
Example #9
| Source File: GradientCheckTestsComputationGraph.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testMultipleOutputsLayer() {
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1))
.updater(new NoOp()).activation(Activation.TANH).graphBuilder().addInputs("i0")
.addLayer("d0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "i0")
.addLayer("d1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "d0")
.addLayer("d2", new DenseLayer.Builder().nIn(2).nOut(2).build(), "d0")
.addLayer("d3", new DenseLayer.Builder().nIn(2).nOut(2).build(), "d0")
.addLayer("out", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(6)
.nOut(2).build(), "d1", "d2", "d3")
.setOutputs("out").build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
int[] minibatchSizes = {1, 3};
for (int mb : minibatchSizes) {
INDArray input = Nd4j.rand(mb, 2);
INDArray out = Nd4j.rand(mb, 2);
String msg = "testMultipleOutputsLayer() - minibatchSize = " + mb;
if (PRINT_RESULTS) {
System.out.println(msg);
// for (int j = 0; j < graph.getNumLayers(); j++)
// System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{out}));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(graph);
}
}
Example #10
| Source File: MultiLayerNeuralNetConfigurationTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testYaml() throws Exception {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
.layer(0, new DenseLayer.Builder().dist(new NormalDistribution(1, 1e-1)).build())
.inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build();
String json = conf.toYaml();
MultiLayerConfiguration from = MultiLayerConfiguration.fromYaml(json);
assertEquals(conf.getConf(0), from.getConf(0));
Properties props = new Properties();
props.put("json", json);
String key = props.getProperty("json");
assertEquals(json, key);
File f = testDir.newFile("props");
f.deleteOnExit();
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f));
props.store(bos, "");
bos.flush();
bos.close();
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f));
Properties props2 = new Properties();
props2.load(bis);
bis.close();
assertEquals(props2.getProperty("json"), props.getProperty("json"));
String yaml = props2.getProperty("json");
MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromYaml(yaml);
assertEquals(conf.getConf(0), conf3.getConf(0));
}
Example #11
| Source File: ComputationGraphTestRNN.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testTruncatedBPTTSimple() {
//Extremely simple test of the 'does it throw an exception' variety
int timeSeriesLength = 12;
int miniBatchSize = 7;
int nIn = 5;
int nOut = 4;
int nTimeSlices = 20;
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder()
.addInputs("in")
.addLayer("0", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
.activation(Activation.TANH)
.dist(new NormalDistribution(0, 0.5)).build(), "in")
.addLayer("1", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
.activation(Activation.TANH)
.dist(new NormalDistribution(0,
0.5))
.build(), "0")
.addLayer("out", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.nIn(8).nOut(nOut)
.activation(Activation.SOFTMAX)
.dist(new NormalDistribution(0, 0.5)).build(), "1")
.setOutputs("out").backpropType(BackpropType.TruncatedBPTT)
.tBPTTBackwardLength(timeSeriesLength).tBPTTForwardLength(timeSeriesLength).build();
Nd4j.getRandom().setSeed(12345);
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
INDArray inputLong = Nd4j.rand(new int[] {miniBatchSize, nIn, nTimeSlices * timeSeriesLength});
INDArray labelsLong = Nd4j.rand(new int[] {miniBatchSize, nOut, nTimeSlices * timeSeriesLength});
graph.fit(new INDArray[] {inputLong}, new INDArray[] {labelsLong});
}
Example #12
| Source File: RegressionTest050.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void regressionTestMLP2() throws Exception {
File f = Resources.asFile("regression_testing/050/050_ModelSerializer_Regression_MLP_2.zip");
MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(f, true);
MultiLayerConfiguration conf = net.getLayerWiseConfigurations();
assertEquals(2, conf.getConfs().size());
DenseLayer l0 = (DenseLayer) conf.getConf(0).getLayer();
assertTrue(l0.getActivationFn() instanceof ActivationLReLU);
assertEquals(3, l0.getNIn());
assertEquals(4, l0.getNOut());
assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInitFn());
assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater());
assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6);
assertEquals(new Dropout(0.6), l0.getIDropout());
assertEquals(0.1, TestUtils.getL1(l0), 1e-6);
assertEquals(new WeightDecay(0.2, false), TestUtils.getWeightDecayReg(l0));
OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer();
assertEquals("identity", l1.getActivationFn().toString());
assertTrue(l1.getLossFn() instanceof LossMSE);
assertEquals(4, l1.getNIn());
assertEquals(5, l1.getNOut());
assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInitFn());
assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getIUpdater());
assertEquals(0.15, ((RmsProp)l1.getIUpdater()).getLearningRate(), 1e-6);
assertEquals(new Dropout(0.6), l1.getIDropout());
assertEquals(0.1, TestUtils.getL1(l1), 1e-6);
assertEquals(new WeightDecay(0.2, false), TestUtils.getWeightDecayReg(l1));
int numParams = (int)net.numParams();
assertEquals(Nd4j.linspace(1, numParams, numParams, Nd4j.dataType()).reshape(1,numParams), net.params());
int updaterSize = (int) new RmsProp().stateSize(numParams);
assertEquals(Nd4j.linspace(1, updaterSize, updaterSize, Nd4j.dataType()).reshape(1,numParams), net.getUpdater().getStateViewArray());
}
Example #13
| Source File: AbstractZooModel.java From wekaDeeplearning4j with GNU General Public License v3.0 | 5 votes |
|
/**
* We need a layer with the correct number of outputs
* @return Default output layer
*/
protected OutputLayer createOutputLayer() {
return new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.nIn(m_numFExtractOutputs).nOut(numLabels)
.weightInit(new NormalDistribution(0, 0.2 * (2.0 / (4096 + numLabels)))) //This weight init dist gave better results than Xavier
.activation(Activation.SOFTMAX).build();
}
Example #14
| Source File: AbstractZooModel.java From wekaDeeplearning4j with GNU General Public License v3.0 | 5 votes |
|
/**
* We need a layer with the correct number of outputs
* @return Default output layer
*/
protected OutputLayer createOutputLayer() {
return new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.nIn(m_numFExtractOutputs).nOut(numLabels)
.weightInit(new NormalDistribution(0, 0.2 * (2.0 / (4096 + numLabels)))) //This weight init dist gave better results than Xavier
.activation(Activation.SOFTMAX).build();
}
Example #15
| Source File: GradientCheckTestsComputationGraph.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testLSTMWithReverseTimeSeriesVertex() {
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf =
new NeuralNetConfiguration.Builder().seed(12345)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1))
.updater(new NoOp()).graphBuilder()
.addInputs("input").setOutputs("out")
.addLayer("lstm_a",
new LSTM.Builder().nIn(2).nOut(3)
.activation(Activation.TANH).build(),
"input")
.addVertex("input_rev", new ReverseTimeSeriesVertex("input"), "input")
.addLayer("lstm_b",
new LSTM.Builder().nIn(2).nOut(3)
.activation(Activation.TANH).build(),
"input_rev")
.addVertex("lstm_b_rev", new ReverseTimeSeriesVertex("input"), "lstm_b")
.addLayer("out", new RnnOutputLayer.Builder().nIn(3 + 3).nOut(2)
.activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT).build(),
"lstm_a", "lstm_b_rev")
.build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
Random r = new Random(12345);
INDArray input = Nd4j.rand(new int[] {2, 2, 4});
INDArray labels = TestUtils.randomOneHotTimeSeries(2, 2, 4);
if (PRINT_RESULTS) {
System.out.println("testLSTMWithReverseTimeSeriesVertex()");
// for (int j = 0; j < graph.getNumLayers(); j++)
// System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{labels}));
String msg = "testLSTMWithDuplicateToTimeSeries()";
assertTrue(msg, gradOK);
//Second: test with input mask arrays.
INDArray inMask = Nd4j.zeros(3, 5);
inMask.putRow(0, Nd4j.create(new double[] {1, 1, 1, 0, 0}));
inMask.putRow(1, Nd4j.create(new double[] {1, 1, 0, 1, 0}));
inMask.putRow(2, Nd4j.create(new double[] {1, 1, 1, 1, 1}));
graph.setLayerMaskArrays(new INDArray[] {inMask}, null);
gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{labels}));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(graph);
}
Example #16
| Source File: CNN1DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnn1DWithLocallyConnected1D() {
Nd4j.getRandom().setSeed(1337);
int[] minibatchSizes = {2, 3};
int length = 7;
int convNIn = 2;
int convNOut1 = 3;
int convNOut2 = 4;
int finalNOut = 4;
int[] kernels = {1};
int stride = 1;
int padding = 0;
Activation[] activations = {Activation.SIGMOID};
for (Activation afn : activations) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < length; j++) {
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp())
.dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
.layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
.build())
.layer(new LocallyConnected1D.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2).hasBias(false)
.build())
.layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.recurrent(convNIn, length)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "Minibatch=" + minibatchSize + ", activationFn="
+ afn + ", kernel = " + kernel;
if (PRINT_RESULTS) {
System.out.println(msg);
// for (int j = 0; j < net.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
}
}
Example #17
| Source File: GradientCheckTestsComputationGraph.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testLSTMWithDuplicateToTimeSeries() {
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf =
new NeuralNetConfiguration.Builder().seed(12345)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1))
.updater(new NoOp()).graphBuilder()
.addInputs("input1", "input2").setOutputs("out")
.addLayer("lstm1",
new LSTM.Builder().nIn(3).nOut(3)
.activation(Activation.TANH).build(),
"input1")
.addLayer("lstm2",
new LSTM.Builder().nIn(2).nOut(4)
.activation(Activation.SOFTSIGN).build(),
"input2")
.addVertex("lastTS", new LastTimeStepVertex("input2"), "lstm2")
.addVertex("duplicate", new DuplicateToTimeSeriesVertex("input2"), "lastTS")
.addLayer("out", new RnnOutputLayer.Builder().nIn(3+4).nOut(2)
.activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT).build(),
"lstm1", "duplicate")
.build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
Random r = new Random(12345);
INDArray input1 = Nd4j.rand(new int[] {2, 3, 4});
INDArray input2 = Nd4j.rand(new int[] {2, 2, 4});
INDArray labels = TestUtils.randomOneHotTimeSeries(2, 2, 4);
if (PRINT_RESULTS) {
System.out.println("testLSTMWithDuplicateToTimeSeries()");
// for (int j = 0; j < graph.getNumLayers(); j++)
// System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input1, input2})
.labels(new INDArray[]{labels}));
String msg = "testLSTMWithDuplicateToTimeSeries()";
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(graph);
}
Example #18
| Source File: CNN1DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnn1DWithZeroPadding1D() {
Nd4j.getRandom().setSeed(1337);
int[] minibatchSizes = {1, 3};
int length = 7;
int convNIn = 2;
int convNOut1 = 3;
int convNOut2 = 4;
int finalNOut = 4;
int[] kernels = {1, 2, 4};
int stride = 1;
int pnorm = 2;
int padding = 0;
int zeroPadding = 2;
int paddedLength = length + 2 * zeroPadding;
Activation[] activations = {Activation.SIGMOID};
SubsamplingLayer.PoolingType[] poolingTypes =
new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};
for (Activation afn : activations) {
for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, paddedLength);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < paddedLength; j++) {
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp())
.dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
.layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
.build())
.layer(new ZeroPadding1DLayer.Builder(zeroPadding).build())
.layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
.build())
.layer(new ZeroPadding1DLayer.Builder(0).build())
.layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
.stride(stride).padding(padding).pnorm(pnorm).build())
.layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.recurrent(convNIn, length)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
+ afn + ", kernel = " + kernel;
if (PRINT_RESULTS) {
System.out.println(msg);
// for (int j = 0; j < net.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
}
}
}
Example #19
| Source File: ComputationGraphTestRNN.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testRnnTimeStep2dInput() {
Nd4j.getRandom().setSeed(12345);
int timeSeriesLength = 6;
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
.addLayer("0", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(5).nOut(7)
.activation(Activation.TANH)
.dist(new NormalDistribution(0, 0.5)).build(), "in")
.addLayer("1", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
.activation(Activation.TANH)
.dist(new NormalDistribution(0,
0.5))
.build(), "0")
.addLayer("2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.nIn(8).nOut(4)
.activation(Activation.SOFTMAX)
.dist(new NormalDistribution(0, 0.5)).build(), "1")
.setOutputs("2").build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
INDArray input3d = Nd4j.rand(new int[] {3, 5, timeSeriesLength});
INDArray out3d = graph.rnnTimeStep(input3d)[0];
assertArrayEquals(out3d.shape(), new long[] {3, 4, timeSeriesLength});
graph.rnnClearPreviousState();
for (int i = 0; i < timeSeriesLength; i++) {
INDArray input2d = input3d.tensorAlongDimension(i, 1, 0);
INDArray out2d = graph.rnnTimeStep(input2d)[0];
assertArrayEquals(out2d.shape(), new long[] {3, 4});
INDArray expOut2d = out3d.tensorAlongDimension(i, 1, 0);
assertEquals(out2d, expOut2d);
}
//Check same but for input of size [3,5,1]. Expect [3,4,1] out
graph.rnnClearPreviousState();
for (int i = 0; i < timeSeriesLength; i++) {
INDArray temp = Nd4j.create(new int[] {3, 5, 1});
temp.tensorAlongDimension(0, 1, 0).assign(input3d.tensorAlongDimension(i, 1, 0));
INDArray out3dSlice = graph.rnnTimeStep(temp)[0];
assertArrayEquals(out3dSlice.shape(), new long[] {3, 4, 1});
assertTrue(out3dSlice.tensorAlongDimension(0, 1, 0).equals(out3d.tensorAlongDimension(i, 1, 0)));
}
}
Example #20
| Source File: GradientCheckTestsComputationGraph.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testLSTMWithLastTimeStepVertex() {
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1))
.updater(new NoOp()).graphBuilder().addInputs("input").setOutputs("out")
.addLayer("lstm1", new LSTM.Builder().nIn(3).nOut(4).activation(Activation.TANH).build(),
"input")
.addVertex("lastTS", new LastTimeStepVertex("input"), "lstm1")
.addLayer("out", new OutputLayer.Builder().nIn(4).nOut(2).activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lastTS")
.build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
Random r = new Random(12345);
INDArray input = Nd4j.rand(new int[] {2, 3, 4});
INDArray labels = TestUtils.randomOneHot(2, 2); //Here: labels are 2d (due to LastTimeStepVertex)
if (PRINT_RESULTS) {
System.out.println("testLSTMWithLastTimeStepVertex()");
// for (int j = 0; j < graph.getNumLayers(); j++)
// System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
}
//First: test with no input mask array
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{labels}));
String msg = "testLSTMWithLastTimeStepVertex()";
assertTrue(msg, gradOK);
//Second: test with input mask arrays.
INDArray inMask = Nd4j.zeros(3, 4);
inMask.putRow(0, Nd4j.create(new double[] {1, 1, 0, 0}));
inMask.putRow(1, Nd4j.create(new double[] {1, 1, 1, 0}));
inMask.putRow(2, Nd4j.create(new double[] {1, 1, 1, 1}));
gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{labels}).inputMask(new INDArray[]{inMask}));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(graph);
}
Example #21
| Source File: CNN1DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnn1DWithCropping1D() {
Nd4j.getRandom().setSeed(1337);
int[] minibatchSizes = {1, 3};
int length = 7;
int convNIn = 2;
int convNOut1 = 3;
int convNOut2 = 4;
int finalNOut = 4;
int[] kernels = {1, 2, 4};
int stride = 1;
int padding = 0;
int cropping = 1;
int croppedLength = length - 2 * cropping;
Activation[] activations = {Activation.SIGMOID};
SubsamplingLayer.PoolingType[] poolingTypes =
new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};
for (Activation afn : activations) {
for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
for (int minibatchSize : minibatchSizes) {
for (int kernel : kernels) {
INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length});
INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, croppedLength);
for (int i = 0; i < minibatchSize; i++) {
for (int j = 0; j < croppedLength; j++) {
labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0);
}
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp())
.dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
.layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
.build())
.layer(new Cropping1D.Builder(cropping).build())
.layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
.stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
.build())
.layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.recurrent(convNIn, length)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
+ afn + ", kernel = " + kernel;
if (PRINT_RESULTS) {
System.out.println(msg);
// for (int j = 0; j < net.getnLayers(); j++)
// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
}
}
}
Example #22
| Source File: TestWeightNoise.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testWeightNoiseConfigJson() {
IWeightNoise[] weightNoises = new IWeightNoise[]{
new DropConnect(0.5),
new DropConnect(new SigmoidSchedule(ScheduleType.ITERATION, 0.5, 0.5, 100)),
new WeightNoise(new NormalDistribution(0, 0.1))
};
for (IWeightNoise wn : weightNoises) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.weightNoise(wn)
.list()
.layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
.layer(new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(new DropConnect(0.25)).build())
.layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
assertEquals(wn, ((BaseLayer) net.getLayer(0).conf().getLayer()).getWeightNoise());
assertEquals(new DropConnect(0.25), ((BaseLayer) net.getLayer(1).conf().getLayer()).getWeightNoise());
assertEquals(wn, ((BaseLayer) net.getLayer(2).conf().getLayer()).getWeightNoise());
TestUtils.testModelSerialization(net);
ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder()
.weightNoise(wn)
.graphBuilder()
.addInputs("in")
.layer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in")
.layer("1", new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(new DropConnect(0.25)).build(), "0")
.layer("2", new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build(), "1")
.setOutputs("2")
.build();
ComputationGraph graph = new ComputationGraph(conf2);
graph.init();
assertEquals(wn, ((BaseLayer) graph.getLayer(0).conf().getLayer()).getWeightNoise());
assertEquals(new DropConnect(0.25), ((BaseLayer) graph.getLayer(1).conf().getLayer()).getWeightNoise());
assertEquals(wn, ((BaseLayer) graph.getLayer(2).conf().getLayer()).getWeightNoise());
TestUtils.testModelSerialization(graph);
graph.fit(new DataSet(Nd4j.create(1,10), Nd4j.create(1,10)));
}
}
Example #23
| Source File: GradientCheckTestsComputationGraph.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnnDepthMerge() {
for(CNN2DFormat format : CNN2DFormat.values()) {
String msg = "testCnnDepthMerge - " + format;
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 0.1))
.updater(new NoOp()).graphBuilder().addInputs("input")
.addLayer("l1", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0)
.nIn(2).nOut(2).activation(Activation.TANH).build(), "input")
.addLayer("l2", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0)
.nIn(2).nOut(2).activation(Activation.TANH).build(), "input")
.addVertex("merge", new MergeVertex(), "l1", "l2")
.addLayer("outputLayer",
new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(5 * 5 * (2 + 2)).nOut(3)
.build(),
"merge")
.setOutputs("outputLayer")
.setInputTypes(InputType.convolutional(6, 6, 2, format))
.build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
Random r = new Random(12345);
INDArray input = Nd4j.rand(DataType.DOUBLE, format == CNN2DFormat.NCHW ? new long[]{5,2,6,6} : new long[]{5,6,6,2});
INDArray labels = Nd4j.zeros(5, 3);
for (int i = 0; i < 5; i++)
labels.putScalar(new int[]{i, r.nextInt(3)}, 1.0);
if (PRINT_RESULTS) {
System.out.println(msg);
// for (int j = 0; j < graph.getNumLayers(); j++)
// System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{labels}));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(graph);
}
}
Example #24
| Source File: GradientCheckTestsComputationGraph.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testBasicIrisWithElementWiseNodeInputSizeGreaterThanTwo() {
ElementWiseVertex.Op[] ops =
new ElementWiseVertex.Op[] {ElementWiseVertex.Op.Add, ElementWiseVertex.Op.Product, ElementWiseVertex.Op.Average, ElementWiseVertex.Op.Max};
for (ElementWiseVertex.Op op : ops) {
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
.dataType(DataType.DOUBLE)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.dist(new NormalDistribution(0, 1))
.updater(new NoOp()).graphBuilder().addInputs("input")
.addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(),
"input")
.addLayer("l2", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.SIGMOID)
.build(), "input")
.addLayer("l3", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(),
"input")
.addVertex("elementwise", new ElementWiseVertex(op), "l1", "l2", "l3")
.addLayer("outputLayer",
new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(5).nOut(3).build(),
"elementwise")
.setOutputs("outputLayer").build();
ComputationGraph graph = new ComputationGraph(conf);
graph.init();
int numParams = (4 * 5 + 5) + (4 * 5 + 5) + (4 * 5 + 5) + (5 * 3 + 3);
assertEquals(numParams, graph.numParams());
Nd4j.getRandom().setSeed(12345);
long nParams = graph.numParams();
INDArray newParams = Nd4j.rand(new long[]{1, nParams});
graph.setParams(newParams);
DataSet ds = new IrisDataSetIterator(150, 150).next();
INDArray min = ds.getFeatures().min(0);
INDArray max = ds.getFeatures().max(0);
ds.getFeatures().subiRowVector(min).diviRowVector(max.sub(min));
INDArray input = ds.getFeatures();
INDArray labels = ds.getLabels();
if (PRINT_RESULTS) {
System.out.println("testBasicIrisWithElementWiseVertex(op=" + op + ")");
// for (int j = 0; j < graph.getNumLayers(); j++)
// System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
.labels(new INDArray[]{labels}));
String msg = "testBasicIrisWithElementWiseVertex(op=" + op + ")";
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(graph);
}
}
Example #25
| Source File: TestConstraints.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testLayerWeightsConstraints() throws Exception {
LayerConstraint[] constraints = new LayerConstraint[]{
new MaxNormConstraint(0.5, 1),
new MinMaxNormConstraint(0.3, 0.4, 1.0, 1),
new NonNegativeConstraint(),
new UnitNormConstraint(1)
};
for (LayerConstraint lc : constraints) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.updater(new Sgd(0.0))
.dist(new NormalDistribution(0, 5))
.list()
.layer(new DenseLayer.Builder().nIn(12).nOut(10)
.constrainWeights(lc).build())
.layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(8).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
LayerConstraint exp = lc.clone();
assertEquals(exp.toString(), net.getLayer(0).conf().getLayer().getConstraints().get(0).toString());
INDArray input = Nd4j.rand(3, 12);
INDArray labels = Nd4j.rand(3, 8);
net.fit(input, labels);
INDArray w0 = net.getParam("0_W");
if (lc instanceof MaxNormConstraint) {
assertTrue(w0.norm2(1).maxNumber().doubleValue() <= 0.5);
} else if (lc instanceof MinMaxNormConstraint) {
assertTrue(w0.norm2(1).minNumber().doubleValue() >= 0.3);
assertTrue(w0.norm2(1).maxNumber().doubleValue() <= 0.4);
} else if (lc instanceof NonNegativeConstraint) {
assertTrue(w0.minNumber().doubleValue() >= 0.0);
} else if (lc instanceof UnitNormConstraint) {
assertEquals(1.0, w0.norm2(1).minNumber().doubleValue(), 1e-6);
assertEquals(1.0, w0.norm2(1).maxNumber().doubleValue(), 1e-6);
}
TestUtils.testModelSerialization(net);
}
}
Example #26
| Source File: CNN3DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testDeconv3d() {
Nd4j.getRandom().setSeed(12345);
// Note: we checked this with a variety of parameters, but it takes a lot of time.
int[] depths = {8, 8, 9};
int[] heights = {8, 9, 9};
int[] widths = {8, 8, 9};
int[][] kernels = {{2, 2, 2}, {3, 3, 3}, {2, 3, 2}};
int[][] strides = {{1, 1, 1}, {1, 1, 1}, {2, 2, 2}};
Activation[] activations = {Activation.SIGMOID, Activation.TANH, Activation.IDENTITY};
ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same, ConvolutionMode.Same};
int[] mbs = {1, 3, 2};
Convolution3D.DataFormat[] dataFormats = new Convolution3D.DataFormat[]{Convolution3D.DataFormat.NCDHW, Convolution3D.DataFormat.NDHWC, Convolution3D.DataFormat.NCDHW};
int convNIn = 2;
int finalNOut = 2;
int[] deconvOut = {2, 3, 4};
for (int i = 0; i < activations.length; i++) {
Activation afn = activations[i];
int miniBatchSize = mbs[i];
int depth = depths[i];
int height = heights[i];
int width = widths[i];
ConvolutionMode mode = modes[i];
int[] kernel = kernels[i];
int[] stride = strides[i];
Convolution3D.DataFormat df = dataFormats[i];
int dOut = deconvOut[i];
INDArray input;
if (df == Convolution3D.DataFormat.NDHWC) {
input = Nd4j.rand(new int[]{miniBatchSize, depth, height, width, convNIn});
} else {
input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width});
}
INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
for (int j = 0; j < miniBatchSize; j++) {
labels.putScalar(new int[]{j, j % finalNOut}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp())
.weightInit(new NormalDistribution(0, 0.1))
.list()
.layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel)
.stride(stride).nIn(convNIn).nOut(dOut).hasBias(false)
.convolutionMode(mode).dataFormat(df)
.build())
.layer(1, new Deconvolution3D.Builder().activation(afn).kernelSize(kernel)
.stride(stride).nOut(dOut).hasBias(false)
.convolutionMode(mode).dataFormat(df)
.build())
.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "DataFormat = " + df + ", minibatch size = " + miniBatchSize + ", activationFn=" + afn
+ ", kernel = " + Arrays.toString(kernel) + ", stride = "
+ Arrays.toString(stride) + ", mode = " + mode.toString()
+ ", input depth " + depth + ", input height " + height
+ ", input width " + width;
if (PRINT_RESULTS) {
log.info(msg);
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
.labels(labels).subset(true).maxPerParam(64));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
Example #27
| Source File: CNN3DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnn3DCropping() {
Nd4j.getRandom().setSeed(42);
int depth = 6;
int height = 6;
int width = 6;
int[] minibatchSizes = {3};
int convNIn = 2;
int convNOut1 = 3;
int convNOut2 = 4;
int denseNOut = 5;
int finalNOut = 8;
int[] kernel = {1, 1, 1};
int[] cropping = {0, 0, 1, 1, 2, 2};
Activation[] activations = {Activation.SIGMOID};
ConvolutionMode[] modes = {ConvolutionMode.Same};
for (Activation afn : activations) {
for (int miniBatchSize : minibatchSizes) {
for (ConvolutionMode mode : modes) {
int outDepth = mode == ConvolutionMode.Same ?
depth : (depth - kernel[0]) + 1;
int outHeight = mode == ConvolutionMode.Same ?
height : (height - kernel[1]) + 1;
int outWidth = mode == ConvolutionMode.Same ?
width : (width - kernel[2]) + 1;
outDepth -= cropping[0] + cropping[1];
outHeight -= cropping[2] + cropping[3];
outWidth -= cropping[4] + cropping[5];
INDArray input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width});
INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
for (int i = 0; i < miniBatchSize; i++) {
labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL)
.dist(new NormalDistribution(0, 1))
.list()
.layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel)
.nIn(convNIn).nOut(convNOut1).hasBias(false)
.convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
.build())
.layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
.nIn(convNOut1).nOut(convNOut2).hasBias(false)
.convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
.build())
.layer(2, new Cropping3D.Builder(cropping).build())
.layer(3, new DenseLayer.Builder().nOut(denseNOut).build())
.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.inputPreProcessor(3,
new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,
convNOut2, true))
.setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
+ ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString()
+ ", input depth " + depth + ", input height " + height
+ ", input width " + width;
if (PRINT_RESULTS) {
log.info(msg);
// for (int j = 0; j < net.getnLayers(); j++) {
// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams());
// }
}
boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS,
DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS,
RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
}
}
Example #28
| Source File: CNN3DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnn3DUpsampling() {
Nd4j.getRandom().setSeed(42);
int depth = 2;
int height = 2;
int width = 2;
int[] minibatchSizes = {3};
int convNIn = 2;
int convNOut = 4;
int denseNOut = 5;
int finalNOut = 42;
int[] upsamplingSize = {2, 2, 2};
Activation[] activations = {Activation.SIGMOID};
ConvolutionMode[] modes = {ConvolutionMode.Truncate};
for (Activation afn : activations) {
for (int miniBatchSize : minibatchSizes) {
for (ConvolutionMode mode : modes) {
for(Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) {
int outDepth = depth * upsamplingSize[0];
int outHeight = height * upsamplingSize[1];
int outWidth = width * upsamplingSize[2];
INDArray input = df == Convolution3D.DataFormat.NCDHW ? Nd4j.rand(miniBatchSize, convNIn, depth, height, width) : Nd4j.rand(miniBatchSize, depth, height, width, convNIn);
INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
for (int i = 0; i < miniBatchSize; i++) {
labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL)
.dist(new NormalDistribution(0, 1))
.seed(12345)
.list()
.layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
.nIn(convNIn).nOut(convNOut).hasBias(false)
.convolutionMode(mode).dataFormat(df)
.build())
.layer(1, new Upsampling3D.Builder(upsamplingSize[0]).dataFormat(df).build())
.layer(2, new DenseLayer.Builder().nOut(denseNOut).build())
.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.inputPreProcessor(2,
new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,
convNOut, true))
.setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
+ ", kernel = " + Arrays.toString(upsamplingSize) + ", mode = " + mode.toString()
+ ", input depth " + depth + ", input height " + height
+ ", input width " + width;
if (PRINT_RESULTS) {
log.info(msg);
// for (int j = 0; j < net.getnLayers(); j++) {
// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams());
// }
}
boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS,
DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS,
RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
}
}
}
Example #29
| Source File: CuDNNGradientChecks.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testBatchNormCnn() throws Exception {
//Note: CuDNN batch norm supports 4d only, as per 5.1 (according to api reference documentation)
Nd4j.getRandom().setSeed(12345);
int minibatch = 10;
int depth = 1;
int hw = 4;
int nOut = 4;
INDArray input = Nd4j.rand(new int[] {minibatch, depth, hw, hw});
INDArray labels = Nd4j.zeros(minibatch, nOut);
Random r = new Random(12345);
for (int i = 0; i < minibatch; i++) {
labels.putScalar(i, r.nextInt(nOut), 1.0);
}
MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
.dataType(DataType.DOUBLE)
.seed(12345L)
.dist(new NormalDistribution(0, 2)).list()
.layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2)
.activation(Activation.IDENTITY).build())
.layer(1, new BatchNormalization.Builder().build())
.layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build())
.layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(nOut).build())
.setInputType(InputType.convolutional(hw, hw, depth));
MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
mln.init();
Field f = org.deeplearning4j.nn.layers.normalization.BatchNormalization.class.getDeclaredField("helper");
f.setAccessible(true);
org.deeplearning4j.nn.layers.normalization.BatchNormalization b =
(org.deeplearning4j.nn.layers.normalization.BatchNormalization) mln.getLayer(1);
BatchNormalizationHelper bn = (BatchNormalizationHelper) f.get(b);
assertTrue(bn instanceof CudnnBatchNormalizationHelper);
//-------------------------------
//For debugging/comparison to no-cudnn case: set helper field to null
// f.set(b, null);
// assertNull(f.get(b));
//-------------------------------
if (PRINT_RESULTS) {
for (int j = 0; j < mln.getnLayers(); j++)
System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
}
//Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
//i.e., runningMean = decay * runningMean + (1-decay) * batchMean
//However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev"));
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, null, false, -1, excludeParams, null);
assertTrue(gradOK);
}
Example #30
| Source File: CNN3DGradientCheckTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testCnn3DZeroPadding() {
Nd4j.getRandom().setSeed(42);
int depth = 4;
int height = 4;
int width = 4;
int[] minibatchSizes = {3};
int convNIn = 2;
int convNOut1 = 3;
int convNOut2 = 4;
int denseNOut = 5;
int finalNOut = 42;
int[] kernel = {2, 2, 2};
int[] zeroPadding = {1, 1, 2, 2, 3, 3};
Activation[] activations = {Activation.SIGMOID};
ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same};
for (Activation afn : activations) {
for (int miniBatchSize : minibatchSizes) {
for (ConvolutionMode mode : modes) {
int outDepth = mode == ConvolutionMode.Same ?
depth : (depth - kernel[0]) + 1;
int outHeight = mode == ConvolutionMode.Same ?
height : (height - kernel[1]) + 1;
int outWidth = mode == ConvolutionMode.Same ?
width : (width - kernel[2]) + 1;
outDepth += zeroPadding[0] + zeroPadding[1];
outHeight += zeroPadding[2] + zeroPadding[3];
outWidth += zeroPadding[4] + zeroPadding[5];
INDArray input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width});
INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut);
for (int i = 0; i < miniBatchSize; i++) {
labels.putScalar(new int[]{i, i % finalNOut}, 1.0);
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL)
.dist(new NormalDistribution(0, 1))
.list()
.layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel)
.nIn(convNIn).nOut(convNOut1).hasBias(false)
.convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
.build())
.layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1)
.nIn(convNOut1).nOut(convNOut2).hasBias(false)
.convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW)
.build())
.layer(2, new ZeroPadding3DLayer.Builder(zeroPadding).build())
.layer(3, new DenseLayer.Builder().nOut(denseNOut).build())
.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(finalNOut).build())
.inputPreProcessor(3,
new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,
convNOut2, true))
.setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build();
String json = conf.toJson();
MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
assertEquals(conf, c2);
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn
+ ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString()
+ ", input depth " + depth + ", input height " + height
+ ", input width " + width;
if (PRINT_RESULTS) {
log.info(msg);
// for (int j = 0; j < net.getnLayers(); j++) {
// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams());
// }
}
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
.labels(labels).subset(true).maxPerParam(512));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
}
}
}
}
