org.deeplearning4j.nn.conf.WorkspaceMode Java Examples
The following examples show how to use
org.deeplearning4j.nn.conf.WorkspaceMode.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: ImageUtils.java From AILibs with GNU Affero General Public License v3.0 | 6 votes |
|
public static PretrainedNNFilter getPretrainedNNFilterByName(final String name, final int layer,
final long[] shape) {
// Thanks to a pointless API requirement, the zoo models require int[] shapes
// while dl4j uses long[] at any other place
final int[] intShape = Arrays.stream(shape).mapToInt(i -> (int) i).toArray();
switch (name) {
case "AlexNet":
return new PretrainedNNFilter(new AlexNet(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
case "LeNet":
return new PretrainedNNFilter(new LeNet(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
case "VGG19":
return new PretrainedNNFilter(new VGG19(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
case "ResNet50":
return new PretrainedNNFilter(new ResNet50(42, intShape, 10, WeightInit.DISTRIBUTION,
new Nesterovs(1e-2, 0.9), CacheMode.NONE, WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer,
shape, name);
default:
return new PretrainedNNFilter(new VGG16(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, "VGG16");
}
}
Example #2
| Source File: TestVAE.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testVaeWeightNoise(){
for(boolean ws : new boolean[]{false, true}) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.seed(12345L)
.trainingWorkspaceMode(ws ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
.inferenceWorkspaceMode(ws ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
.weightNoise(new WeightNoise(new org.deeplearning4j.nn.conf.distribution.NormalDistribution(0.1, 0.3)))
.list().layer(0,
new VariationalAutoencoder.Builder().nIn(10).nOut(3)
.encoderLayerSizes(5).decoderLayerSizes(6)
.pzxActivationFunction(Activation.TANH)
.reconstructionDistribution(new GaussianReconstructionDistribution())
.activation(new ActivationTanH())
.build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray arr = Nd4j.rand(3, 10);
net.pretrainLayer(0, arr);
}
}
Example #3
| Source File: BarnesHutTsne.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter,
double realMin, double initialMomentum, double finalMomentum, double momentum,
int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance,
double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener,
double minGain,int vpTreeWorkers) {
this(numDimensions, simiarlityFunction, theta, invert, maxIter, realMin, initialMomentum, finalMomentum,
momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, learningRate,
useAdaGrad, perplexity, TrainingListener, minGain, vpTreeWorkers, WorkspaceMode.NONE, null);
}
Example #4
| Source File: BarnesHutTsne.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter,
double realMin, double initialMomentum, double finalMomentum, double momentum,
int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance,
double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener,
double minGain,int vpTreeWorkers, WorkspaceMode workspaceMode, INDArray staticInput) {
this.maxIter = maxIter;
this.realMin = realMin;
this.initialMomentum = initialMomentum;
this.finalMomentum = finalMomentum;
this.momentum = momentum;
this.normalize = normalize;
this.useAdaGrad = useAdaGrad;
this.stopLyingIteration = stopLyingIteration;
this.learningRate = learningRate;
this.switchMomentumIteration = switchMomentumIteration;
this.tolerance = tolerance;
this.perplexity = perplexity;
this.minGain = minGain;
this.numDimensions = numDimensions;
this.simiarlityFunction = simiarlityFunction;
this.theta = theta;
this.trainingListener = TrainingListener;
this.invert = invert;
this.vpTreeWorkers = vpTreeWorkers;
this.workspaceMode = workspaceMode;
if(this.workspaceMode == null)
this.workspaceMode = WorkspaceMode.NONE;
initializer = (staticInput != null) ? new Initializer(staticInput) : new Initializer();
}
Example #5
| Source File: SymmetricTrainer.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
public SymmetricTrainer(@NonNull Model originalModel, String uuid, int threadIdx, @NonNull WorkspaceMode mode,
@NonNull ParallelWrapper wrapper, boolean useMDS) {
super();
this.uuid = uuid + "_thread_" + threadIdx;
this.useMDS = useMDS;
this.originalModel = originalModel;
this.threadId = threadIdx;
this.workspaceMode = mode;
this.parallelWrapper = wrapper;
this.accumulator = wrapper.getGradientsAccumulator();
}
Example #6
| Source File: DefaultTrainerContext.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
/**
* Create a {@link Trainer}
* based on the given parameters
*
* @param threadId the thread id to use for this worker
* @param model the model to start the trainer with
* @param rootDevice the root device id
* @param useMDS whether to use MultiDataSet or DataSet
* or not
* @param wrapper the wrapper instance to use with this trainer (this refernece is needed
* for coordination with the {@link ParallelWrapper} 's {@link TrainingListener}
* @return the created training instance
*/
@Override
public Trainer create(String uuid, int threadId, Model model, int rootDevice, boolean useMDS, ParallelWrapper wrapper,
WorkspaceMode mode, int averagingFrequency) {
DefaultTrainer trainer = DefaultTrainer.builder().originalModel(model).replicatedModel(model).threadId(threadId)
.parallelWrapper(wrapper).workspaceMode(mode).useMDS(useMDS)
.uuid(uuid + "_thread_" + threadId)
.averagingFrequency(averagingFrequency).build();
trainer.setName("DefaultTrainer thread " + threadId);
trainer.setDaemon(true);
return trainer;
}
Example #7
| Source File: TestSameDiffLambda.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testSameDiffLamdaVertexBasic(){
for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
log.info("--- Workspace Mode: {} ---", wsm);
Nd4j.getRandom().setSeed(12345);
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
.trainingWorkspaceMode(wsm)
.inferenceWorkspaceMode(wsm)
.dataType(DataType.DOUBLE)
.seed(12345)
.updater(new Adam(0.01))
.graphBuilder()
.addInputs("in1", "in2")
.addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
.addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
.addVertex("lambda", new SameDiffSimpleLambdaVertex(), "0", "1")
.addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lambda")
.setOutputs("2")
.build();
//Equavalent, not using SameDiff Lambda:
ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
.trainingWorkspaceMode(wsm)
.inferenceWorkspaceMode(wsm)
.dataType(DataType.DOUBLE)
.seed(12345)
.updater(new Adam(0.01))
.graphBuilder()
.addInputs("in1", "in2")
.addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
.addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
.addVertex("elementwise", new ElementWiseVertex(ElementWiseVertex.Op.Product), "0", "1")
.addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT).build(), "elementwise")
.setOutputs("3")
.build();
ComputationGraph lambda = new ComputationGraph(conf);
lambda.init();
ComputationGraph std = new ComputationGraph(confStd);
std.init();
lambda.setParams(std.params());
INDArray in1 = Nd4j.rand(3, 5);
INDArray in2 = Nd4j.rand(3, 5);
INDArray labels = TestUtils.randomOneHot(3, 5);
MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{in1, in2}, new INDArray[]{labels});
INDArray outLambda = lambda.output(in1, in2)[0];
INDArray outStd = std.output(in1, in2)[0];
assertEquals(outLambda, outStd);
double scoreLambda = lambda.score(mds);
double scoreStd = std.score(mds);
assertEquals(scoreStd, scoreLambda, 1e-6);
for (int i = 0; i < 3; i++) {
lambda.fit(mds);
std.fit(mds);
String s = String.valueOf(i);
assertEquals(s, std.params(), lambda.params());
assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
}
ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
outLambda = loaded.output(in1, in2)[0];
outStd = std.output(in1, in2)[0];
assertEquals(outStd, outLambda);
//Sanity check on different minibatch sizes:
INDArray newIn1 = Nd4j.vstack(in1, in1);
INDArray newIn2 = Nd4j.vstack(in2, in2);
INDArray outMbsd = lambda.output(newIn1, newIn2)[0];
INDArray outMb = std.output(newIn1, newIn2)[0];
assertEquals(outMb, outMbsd);
}
}
Example #8
| Source File: TestScoreFunctions.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testROCScoreFunctions() throws Exception {
for (boolean auc : new boolean[]{true, false}) {
for (ROCScoreFunction.ROCType rocType : ROCScoreFunction.ROCType.values()) {
String msg = (auc ? "AUC" : "AUPRC") + " - " + rocType;
log.info("Starting: " + msg);
ParameterSpace<Double> lr = new ContinuousParameterSpace(1e-5, 1e-3);
int nOut = (rocType == ROCScoreFunction.ROCType.ROC ? 2 : 10);
LossFunctions.LossFunction lf = (rocType == ROCScoreFunction.ROCType.BINARY ?
LossFunctions.LossFunction.XENT : LossFunctions.LossFunction.MCXENT);
Activation a = (rocType == ROCScoreFunction.ROCType.BINARY ? Activation.SIGMOID : Activation.SOFTMAX);
MultiLayerSpace mls = new MultiLayerSpace.Builder()
.trainingWorkspaceMode(WorkspaceMode.NONE)
.inferenceWorkspaceMode(WorkspaceMode.NONE)
.updater(new AdamSpace(lr))
.weightInit(WeightInit.XAVIER)
.layer(new OutputLayerSpace.Builder().nIn(784).nOut(nOut)
.activation(a)
.lossFunction(lf).build())
.build();
CandidateGenerator cg = new RandomSearchGenerator(mls);
ResultSaver rs = new InMemoryResultSaver();
ScoreFunction sf = new ROCScoreFunction(rocType, (auc ? ROCScoreFunction.Metric.AUC : ROCScoreFunction.Metric.AUPRC));
OptimizationConfiguration oc = new OptimizationConfiguration.Builder()
.candidateGenerator(cg)
.dataProvider(new DP(rocType))
.modelSaver(rs)
.scoreFunction(sf)
.terminationConditions(new MaxCandidatesCondition(3))
.rngSeed(12345)
.build();
IOptimizationRunner runner = new LocalOptimizationRunner(oc, new MultiLayerNetworkTaskCreator());
runner.execute();
List<ResultReference> list = runner.getResults();
for (ResultReference rr : list) {
DataSetIterator testIter = new MnistDataSetIterator(4, 16, false, false, false, 12345);
testIter.setPreProcessor(new PreProc(rocType));
OptimizationResult or = rr.getResult();
MultiLayerNetwork net = (MultiLayerNetwork) or.getResultReference().getResultModel();
double expScore;
switch (rocType){
case ROC:
if(auc){
expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUC();
} else {
expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUCPR();
}
break;
case BINARY:
if(auc){
expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAuc();
} else {
expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAUCPR();
}
break;
case MULTICLASS:
if(auc){
expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUC();
} else {
expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUCPR();
}
break;
default:
throw new RuntimeException();
}
DataSetIterator iter = new MnistDataSetIterator(4, 16, false, false, false, 12345);
iter.setPreProcessor(new PreProc(rocType));
assertEquals(msg, expScore, or.getScore(), 1e-4);
}
}
}
}
Example #9
| Source File: TestGraphNodes.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testLastTimeStepWithTransfer(){
int lstmLayerSize = 16;
int numLabelClasses = 10;
int numInputs = 5;
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
.trainingWorkspaceMode(WorkspaceMode.NONE)
.inferenceWorkspaceMode(WorkspaceMode.NONE)
.seed(123) //Random number generator seed for improved repeatability. Optional.
.updater(new AdaDelta())
.weightInit(WeightInit.XAVIER)
.graphBuilder()
.addInputs("rr")
.setInputTypes(InputType.recurrent(30))
.addLayer("1", new GravesLSTM.Builder().activation(Activation.TANH).nIn(numInputs).nOut(lstmLayerSize).dropOut(0.9).build(), "rr")
.addLayer("2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nOut(numLabelClasses).build(), "1")
.setOutputs("2")
.build();
ComputationGraph net = new ComputationGraph(conf);
net.init();
ComputationGraph updatedModel = new TransferLearning.GraphBuilder(net)
.addVertex("laststepoutput", new LastTimeStepVertex("rr"), "2")
.setOutputs("laststepoutput")
.build();
INDArray input = Nd4j.rand(new int[]{10, numInputs, 16});
INDArray[] out = updatedModel.output(input);
assertNotNull(out);
assertEquals(1, out.length);
assertNotNull(out[0]);
assertArrayEquals(new long[]{10, numLabelClasses}, out[0].shape());
Map<String,INDArray> acts = updatedModel.feedForward(input, false);
assertEquals(4, acts.size()); //2 layers + input + vertex output
assertNotNull(acts.get("laststepoutput"));
assertArrayEquals(new long[]{10, numLabelClasses}, acts.get("laststepoutput").shape());
String toString = out[0].toString();
}
Example #10
| Source File: TestCompGraphUnsupervised.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void compareImplementations() throws Exception {
for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {
MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
.seed(12345)
.updater(new Adam(1e-3))
.weightInit(WeightInit.XAVIER)
.inferenceWorkspaceMode(wsm)
.trainingWorkspaceMode(wsm)
.list()
.layer(new VariationalAutoencoder.Builder()
.nIn(784)
.nOut(32)
.encoderLayerSizes(16)
.decoderLayerSizes(16)
.activation(Activation.TANH)
.pzxActivationFunction(Activation.SIGMOID)
.reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID))
.build())
.layer(new VariationalAutoencoder.Builder()
.nIn(32)
.nOut(8)
.encoderLayerSizes(16)
.decoderLayerSizes(16)
.activation(Activation.TANH)
.pzxActivationFunction(Activation.SIGMOID)
.reconstructionDistribution(new GaussianReconstructionDistribution(Activation.TANH))
.build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf2);
net.init();
ComputationGraph cg = net.toComputationGraph();
cg.getConfiguration().setInferenceWorkspaceMode(wsm);
cg.getConfiguration().setTrainingWorkspaceMode(wsm);
DataSetIterator ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
Nd4j.getRandom().setSeed(12345);
net.pretrainLayer(0, ds);
ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
Nd4j.getRandom().setSeed(12345);
cg.pretrainLayer("0", ds);
assertEquals(net.params(), cg.params());
}
}
Example #11
| Source File: ValidateCuDNN.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public static void validateLayers(MultiLayerNetwork net, List<Class<?>> classesToTest, boolean testAllCudnnPresent, int[] fShape, int[] lShape, double maxRE, double minAbsErr) {
for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {
net.getLayerWiseConfigurations().setTrainingWorkspaceMode(wsm);
net.getLayerWiseConfigurations().setInferenceWorkspaceMode(wsm);
Nd4j.getRandom().setSeed(12345);
INDArray features = Nd4j.rand(fShape);
INDArray labels = Nd4j.rand(lShape);
labels = Nd4j.exec(new IsMax(labels, 1))[0].castTo(features.dataType());
List<CuDNNValidationUtil.TestCase> testCaseList = new ArrayList<>();
List<DataSet> dataSets = new ArrayList<>();
for (int i = 0; i < 6; i++) {
INDArray f = Nd4j.rand(fShape);
INDArray l = Nd4j.rand(lShape);
l = Nd4j.exec(new IsMax(l, 1))[0].castTo(features.dataType());
dataSets.add(new DataSet(f, l));
}
DataSetIterator iter = new ExistingDataSetIterator(dataSets);
for (Class<?> c : classesToTest) {
String name = "WS=" + wsm + ", testCudnnFor=" + c.getSimpleName();
testCaseList.add(CuDNNValidationUtil.TestCase.builder()
.testName(name)
.allowCudnnHelpersForClasses(Collections.<Class<?>>singletonList(c))
.testForward(true)
.testScore(true)
.testBackward(true)
.testTraining(true)
.trainFirst(false)
.features(features)
.labels(labels)
.data(iter)
.maxRE(maxRE)
.minAbsErr(minAbsErr)
.build());
}
if(testAllCudnnPresent) {
testCaseList.add(CuDNNValidationUtil.TestCase.builder()
.testName("WS=" + wsm + ", ALL CLASSES")
.allowCudnnHelpersForClasses(classesToTest)
.testForward(true)
.testScore(true)
.testBackward(true)
.trainFirst(false)
.features(features)
.labels(labels)
.data(iter)
.maxRE(maxRE)
.minAbsErr(minAbsErr)
.build());
}
for (CuDNNValidationUtil.TestCase tc : testCaseList) {
log.info("Running test: " + tc.getTestName());
CuDNNValidationUtil.validateMLN(net, tc);
}
}
}
Example #12
| Source File: SymmetricTrainerContextTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testEqualUuid1() {
MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
.l2(0.0005)
//.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
.weightInit(WeightInit.XAVIER)
.updater(new Nesterovs(0.01, 0.9)).list()
.layer(0, new ConvolutionLayer.Builder(5, 5)
//nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
.nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
.layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
.stride(2, 2).build())
.layer(2, new ConvolutionLayer.Builder(5, 5)
//Note that nIn needed be specified in later layers
.stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
.layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
.stride(2, 2).build())
.layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
.layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.nOut(outputNum).activation(Activation.SOFTMAX).build())
.setInputType(InputType.convolutionalFlat(28, 28, nChannels));
MultiLayerConfiguration conf = builder.build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
// ParallelWrapper will take care of load balancing between GPUs.
ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
// DataSets prefetching options. Set this value with respect to number of actual devices
.prefetchBuffer(24)
// set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
.workers(2)
// rare averaging improves performance, but might reduce model accuracy
.averagingFrequency(3)
// if set to TRUE, on every averaging model score will be reported
.reportScoreAfterAveraging(true)
// optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
.build();
val trainer = new SymmetricTrainer(model, "alpha", 3, WorkspaceMode.NONE, wrapper, true);
assertEquals("alpha_thread_3", trainer.getUuid());
}
Example #13
| Source File: DefaultTrainerContextTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testEqualUuid1() {
MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
.l2(0.0005)
//.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
.weightInit(WeightInit.XAVIER)
.updater(new Nesterovs(0.01, 0.9)).list()
.layer(0, new ConvolutionLayer.Builder(5, 5)
//nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
.nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
.layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
.stride(2, 2).build())
.layer(2, new ConvolutionLayer.Builder(5, 5)
//Note that nIn needed be specified in later layers
.stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
.layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
.stride(2, 2).build())
.layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
.layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.nOut(outputNum).activation(Activation.SOFTMAX).build())
.setInputType(InputType.convolutionalFlat(28, 28, nChannels));
MultiLayerConfiguration conf = builder.build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
// ParallelWrapper will take care of load balancing between GPUs.
ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
// DataSets prefetching options. Set this value with respect to number of actual devices
.prefetchBuffer(24)
// set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
.workers(2)
// rare averaging improves performance, but might reduce model accuracy
.averagingFrequency(3)
// if set to TRUE, on every averaging model score will be reported
.reportScoreAfterAveraging(true)
// optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
.build();
val context = new DefaultTrainerContext();
val trainer = context.create("alpha", 3, model, 0, true, wrapper, WorkspaceMode.NONE, 3);
assertEquals("alpha_thread_3", trainer.getUuid());
}
Example #14
| Source File: MultiLayerSpace.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public Builder trainingWorkspaceMode(WorkspaceMode workspaceMode){
this.trainingWorkspaceMode = workspaceMode;
return this;
}
Example #15
| Source File: MultiLayerSpace.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public Builder inferenceWorkspaceMode(WorkspaceMode workspaceMode){
this.inferenceWorkspaceMode = workspaceMode;
return this;
}
Example #16
| Source File: ComputationGraphSpace.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public Builder trainingWorkspaceMode(WorkspaceMode workspaceMode){
this.trainingWorkspaceMode = workspaceMode;
return this;
}
Example #17
| Source File: ComputationGraphSpace.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public Builder inferenceWorkspaceMode(WorkspaceMode workspaceMode){
this.inferenceWorkspaceMode = workspaceMode;
return this;
}
Example #18
| Source File: ResNetwork.java From FancyBing with GNU General Public License v3.0 | 4 votes |
|
public ComputationGraph init() {
ComputationGraphConfiguration.GraphBuilder graph = new NeuralNetConfiguration.Builder()
.seed(seed)
.iterations(iterations)
.activation(Activation.LEAKYRELU)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
.lrPolicyDecayRate(0.5)
.learningRateDecayPolicy(LearningRatePolicy.Score)
.updater(Adam.builder().build())
.weightInit(WeightInit.XAVIER)
.learningRate(0.02)
.miniBatch(miniBatch)
.convolutionMode(ConvolutionMode.Truncate)
.trainingWorkspaceMode(WorkspaceMode.SINGLE)
.inferenceWorkspaceMode(WorkspaceMode.SINGLE)
.graphBuilder();
// set input & output
graph
.addInputs("input").setInputTypes(InputType.convolutionalFlat(height, width, channels))
.addLayer("policy", new OutputLayer.Builder()
.lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.activation(Activation.SOFTMAX)
.nOut(numClasses).build(), "embeddings_c")
.addLayer("value1", new OutputLayer.Builder()
.lossFunction(LossFunctions.LossFunction.MSE)
.activation(Activation.TANH)
.nOut(1).build(), "embeddings_r1")
// .addLayer("value2", new OutputLayer.Builder()
// .lossFunction(LossFunctions.LossFunction.MSE)
// .activation(Activation.TANH)
// .nOut(1).build(), "embeddings_r2")
.setOutputs("policy", "value1", "value2")
.backprop(true).pretrain(false);
int kernelSize = 128;
graph.addLayer("c-layer0", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.LEAKYRELU).nOut(kernelSize).build(), "input");
int blockNum = 8;
String prevLayer = "c-layer0";
for (int i = 1; i <= blockNum; i++) {
String layerName = "c-block" + i + "-";
graph.addLayer(layerName + "1", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.LEAKYRELU).nOut(kernelSize).build(), prevLayer);
graph.addLayer(layerName + "2", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(kernelSize).build(), layerName + "1");
graph.addVertex("shortcut" + i, new ElementWiseVertex(ElementWiseVertex.Op.Add), layerName + "2", prevLayer);
graph.addLayer(layerName + "3", new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build(), "shortcut" + i);
prevLayer = layerName + "3";
}
// for classification
graph.addLayer("embeddings_c", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(2).build(), prevLayer);
// for value regression
graph.addLayer("reg-c-layer1", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(1).build(), prevLayer);
graph.addLayer("embeddings_r1", new DenseLayer.Builder().activation(Activation.IDENTITY).nOut(256).build(), "reg-c-layer1");
// graph.addLayer("reg-c-layer2", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(1).build(), prevLayer);
// graph.addLayer("embeddings_r2", new DenseLayer.Builder().activation(Activation.IDENTITY).nOut(256).build(), "reg-c-layer2");
ComputationGraphConfiguration conf = graph.build();
ComputationGraph model = new ComputationGraph(conf);
model.init();
log.info("\nNumber of params: " + model.numParams()+"\n");
return model;
}
Example #19
| Source File: TestSameDiffConv.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testSameDiffConvGradient() {
int imgH = 8;
int imgW = 8;
int nIn = 3;
int nOut = 4;
int[] kernel = {2, 2};
int[] strides = {1, 1};
int[] dilation = {1, 1};
int count = 0;
//Note: to avoid the exporential number of tests here, we'll randomly run every Nth test only.
//With n=1, m=3 this is 1 out of every 3 tests (on average)
Random r = new Random(12345);
int n = 1;
int m = 5;
for(boolean workspaces : new boolean[]{false, true}) {
for (int minibatch : new int[]{5, 1}) {
for (boolean hasBias : new boolean[]{true, false}) {
for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) {
int i = r.nextInt(m);
if (i >= n) {
//Example: n=2, m=3... skip on i=2, run test on i=0, i=1
continue;
}
String msg = "Test " + (count++) + " - minibatch=" + minibatch + ", ConvolutionMode=" + cm + ", hasBias=" + hasBias;
int outH = cm == ConvolutionMode.Same ? imgH : (imgH-2);
int outW = cm == ConvolutionMode.Same ? imgW : (imgW-2);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.dataType(DataType.DOUBLE)
.seed(12345)
.updater(new NoOp())
.trainingWorkspaceMode(workspaces ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
.inferenceWorkspaceMode(workspaces ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
.list()
.layer(new SameDiffConv.Builder()
.weightInit(WeightInit.XAVIER)
.nIn(nIn)
.nOut(nOut)
.kernelSize(kernel)
.stride(strides)
.dilation(dilation)
.convolutionMode(cm)
.activation(Activation.TANH)
.hasBias(hasBias)
.build())
.layer(new SameDiffConv.Builder()
.weightInit(WeightInit.XAVIER)
.nIn(nOut)
.nOut(nOut)
.kernelSize(kernel)
.stride(strides)
.dilation(dilation)
.convolutionMode(cm)
.activation(Activation.SIGMOID)
.hasBias(hasBias)
.build())
.layer(new OutputLayer.Builder().activation(Activation.SOFTMAX)
.lossFunction(LossFunctions.LossFunction.MCXENT)
.nIn(nOut * outH * outW)
.nOut(nOut).build())
.inputPreProcessor(2, new CnnToFeedForwardPreProcessor(outH, outW, nOut))
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray f = Nd4j.rand(new int[]{minibatch, nIn, imgH, imgW});
INDArray l = TestUtils.randomOneHot(minibatch, nOut);
log.info("Starting: " + msg);
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(f)
.labels(l).subset(true).maxPerParam(50));
assertTrue(msg, gradOK);
TestUtils.testModelSerialization(net);
//Sanity check on different minibatch sizes:
INDArray newIn = Nd4j.vstack(f, f);
net.output(newIn);
}
}
}
}
}
Example #20
| Source File: TrainCifar10Model.java From Java-Machine-Learning-for-Computer-Vision with MIT License | 4 votes |
|
private void train() throws IOException {
ZooModel zooModel = VGG16.builder().build();
ComputationGraph vgg16 = (ComputationGraph) zooModel.initPretrained(PretrainedType.CIFAR10);
log.info(vgg16.summary());
IUpdater iUpdaterWithDefaultConfig = Updater.ADAM.getIUpdaterWithDefaultConfig();
iUpdaterWithDefaultConfig.setLrAndSchedule(0.1, null);
FineTuneConfiguration fineTuneConf = new FineTuneConfiguration.Builder()
.seed(1234)
// .weightInit(WeightInit.XAVIER)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.activation(Activation.RELU)
.updater(iUpdaterWithDefaultConfig)
.cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
.miniBatch(true)
.inferenceWorkspaceMode(WorkspaceMode.ENABLED)
.trainingWorkspaceMode(WorkspaceMode.ENABLED)
.pretrain(true)
.backprop(true)
.build();
ComputationGraph cifar10 = new TransferLearning.GraphBuilder(vgg16)
.setWorkspaceMode(WorkspaceMode.ENABLED)
.fineTuneConfiguration(fineTuneConf)
.setInputTypes(InputType.convolutionalFlat(ImageUtils.HEIGHT,
ImageUtils.WIDTH, 3))
.removeVertexAndConnections("dense_2_loss")
.removeVertexAndConnections("dense_2")
.removeVertexAndConnections("dense_1")
.removeVertexAndConnections("dropout_1")
.removeVertexAndConnections("embeddings")
.removeVertexAndConnections("flatten_1")
.addLayer("dense_1", new DenseLayer.Builder()
.nIn(4096)
.nOut(EMBEDDINGS)
.activation(Activation.RELU).build(), "block3_pool")
.addVertex("embeddings", new L2NormalizeVertex(new int[]{}, 1e-12), "dense_1")
.addLayer("lossLayer", new CenterLossOutputLayer.Builder()
.lossFunction(LossFunctions.LossFunction.SQUARED_LOSS)
.activation(Activation.SOFTMAX).nIn(EMBEDDINGS).nOut(NUM_POSSIBLE_LABELS)
.lambda(LAMBDA).alpha(0.9)
.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).build(),
"embeddings")
.setOutputs("lossLayer")
.build();
log.info(cifar10.summary());
File rootDir = new File("CarTracking/train_from_video_" + NUM_POSSIBLE_LABELS);
DataSetIterator dataSetIterator = ImageUtils.createDataSetIterator(rootDir, NUM_POSSIBLE_LABELS, BATCH_SIZE);
DataSetIterator testIterator = ImageUtils.createDataSetIterator(rootDir, NUM_POSSIBLE_LABELS, BATCH_SIZE);
cifar10.setListeners(new ScoreIterationListener(2));
int iEpoch = I_EPOCH;
while (iEpoch < EPOCH_TRAINING) {
while (dataSetIterator.hasNext()) {
DataSet trainMiniBatchData = null;
try {
trainMiniBatchData = dataSetIterator.next();
} catch (Exception e) {
e.printStackTrace();
}
cifar10.fit(trainMiniBatchData);
}
iEpoch++;
String modelName = PREFIX + NUM_POSSIBLE_LABELS + "_epoch_data_e" + EMBEDDINGS + "_b" + BATCH_SIZE + "_" + iEpoch + ".zip";
saveProgress(cifar10, iEpoch, modelName);
testResults(cifar10, testIterator, iEpoch, modelName);
dataSetIterator.reset();
log.info("iEpoch = " + iEpoch);
}
}
Example #21
| Source File: Vasttext.java From scava with Eclipse Public License 2.0 | 4 votes |
|
private ComputationGraph VasttextTextualAndNumeric()
{
Activation activation = null;
LossFunction loss = null;
//If multilabel, it is considered according to the book "Deep Learning with Python" to use the following parameters
if(multiLabel)
{
activation = Activation.SIGMOID;
loss = LossFunction.XENT; //Binary Crossentropy
}
else
{
//We're using a softmax/cross entropy for the binary classification, as the number of neurons is two. If the number of neurons would be one, then
//the activation would be sigmoid and the loss binary crossentropy
activation = Activation.SOFTMAX;
loss = LossFunction.MCXENT; //CATEGORICAL_CROSSENTROPY
}
System.err.println("LR:"+lr);
System.err.println("Dense:"+denseDimension);
ComputationGraphConfiguration nnConf = new NeuralNetConfiguration.Builder()
.updater(new Adam(lr))
.weightInit(WeightInit.XAVIER)
.trainingWorkspaceMode(WorkspaceMode.ENABLED)
.inferenceWorkspaceMode(WorkspaceMode.ENABLED)
.graphBuilder()
.addInputs("Text", "Extra")
//Embeddings Parts
.addLayer("Embeddings", new EmbeddingSequenceLayer.Builder()
.nIn(textFeaturesSize)
.nOut(denseDimension)
.activation(Activation.IDENTITY)
//.activation(Activation.TANH)
//.dropOut(0.0)
.build(), "Text")
.addLayer("GlobalPooling", new GlobalPoolingLayer.Builder()
.poolingType(PoolingType.AVG)
.poolingDimensions(2)
.collapseDimensions(true)
//.dropOut(0.0)
.build(), "Embeddings")
//We're merging directly the values from the extra
.addVertex("Merge", new MergeVertex(), "GlobalPooling","Extra")
.addLayer("DenseAll", new DenseLayer.Builder()
.nIn(denseDimension+numericFeaturesSize)
.nOut(denseDimension/2)
//.dropOut(0.5)
//.l2(0.001)
.build(), "Merge")
.addLayer("Output", new OutputLayer.Builder()
//.dropOut(0.5)
.nIn(denseDimension/2)
.nOut(labelsSize)
.activation(activation)
.lossFunction(loss)
.build(), "DenseAll")
.setOutputs("Output")
.pretrain(false)
.backprop(true)
.build();
return new ComputationGraph(nnConf);
}
Example #22
| Source File: Vasttext.java From scava with Eclipse Public License 2.0 | 4 votes |
|
private MultiLayerNetwork VasttextTextual()
{
Activation activation = null;
LossFunction loss = null;
//If multilabel, it is considered according to the book "Deep Learning with Python" to use the following parameters
if(multiLabel)
{
activation = Activation.SIGMOID;
loss = LossFunction.XENT; //Binary Crossentropy
}
else
{
//We're using a softmax/cross entropy for the binary classification, as the number of neurons is two. If the number of neurons would be one, then
//the activation would be sigmoid and the loss binary crossentropy
activation = Activation.SOFTMAX;
loss = LossFunction.MCXENT; //CATEGORICAL_CROSSENTROPY
}
MultiLayerConfiguration nnConf = new NeuralNetConfiguration.Builder()
.updater(new Adam(lr))
.weightInit(WeightInit.XAVIER)
.trainingWorkspaceMode(WorkspaceMode.ENABLED)
.inferenceWorkspaceMode(WorkspaceMode.ENABLED)
.list()
.layer(0, new EmbeddingSequenceLayer.Builder()
.nIn(textFeaturesSize)
.nOut(denseDimension)
.activation(Activation.IDENTITY)
.build())
.layer(1, new GlobalPoolingLayer.Builder()
.poolingType(PoolingType.AVG)
.poolingDimensions(2)
.collapseDimensions(true)
.build())
.layer(2, new OutputLayer.Builder()
.nIn(denseDimension)
.nOut(labelsSize)
.activation(activation)
.lossFunction(loss)
.build())
.pretrain(false).backprop(true).build();
return new MultiLayerNetwork(nnConf);
}
Example #23
| Source File: TsneTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testSimple() throws Exception {
//Simple sanity check
for( int test=0; test <=1; test++){
boolean syntheticData = test == 1;
WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);
//STEP 1: Initialization
int iterations = 50;
//create an n-dimensional array of doubles
Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words
//STEP 2: Turn text input into a list of words
INDArray weights;
if(syntheticData){
weights = Nd4j.rand(250, 200);
} else {
log.info("Load & Vectorize data....");
File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile(); //Open the file
//Get the data of all unique word vectors
Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
VocabCache cache = vectors.getSecond();
weights = vectors.getFirst().getSyn0(); //seperate weights of unique words into their own list
for (int i = 0; i < cache.numWords(); i++) //seperate strings of words into their own list
cacheList.add(cache.wordAtIndex(i));
}
//STEP 3: build a dual-tree tsne to use later
log.info("Build model....");
BarnesHutTsne tsne = new BarnesHutTsne.Builder()
.setMaxIter(iterations)
.theta(0.5)
.normalize(false)
.learningRate(500)
.useAdaGrad(false)
.workspaceMode(wsm)
.build();
//STEP 4: establish the tsne values and save them to a file
log.info("Store TSNE Coordinates for Plotting....");
File outDir = testDir.newFolder();
tsne.fit(weights);
tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
}
}
Example #24
| Source File: TsneTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testPerformance() throws Exception {
StopWatch watch = new StopWatch();
watch.start();
for( int test=0; test <=1; test++){
boolean syntheticData = test == 1;
WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);
//STEP 1: Initialization
int iterations = 50;
//create an n-dimensional array of doubles
Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words
//STEP 2: Turn text input into a list of words
INDArray weights;
if(syntheticData){
weights = Nd4j.rand(DataType.FLOAT, 250, 20);
} else {
log.info("Load & Vectorize data....");
File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile(); //Open the file
//Get the data of all unique word vectors
Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
VocabCache cache = vectors.getSecond();
weights = vectors.getFirst().getSyn0(); //seperate weights of unique words into their own list
for (int i = 0; i < cache.numWords(); i++) //seperate strings of words into their own list
cacheList.add(cache.wordAtIndex(i));
}
//STEP 3: build a dual-tree tsne to use later
log.info("Build model....");
BarnesHutTsne tsne = new BarnesHutTsne.Builder()
.setMaxIter(iterations)
.theta(0.5)
.normalize(false)
.learningRate(500)
.useAdaGrad(false)
.workspaceMode(wsm)
.build();
//STEP 4: establish the tsne values and save them to a file
log.info("Store TSNE Coordinates for Plotting....");
File outDir = testDir.newFolder();
tsne.fit(weights);
tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
}
watch.stop();
System.out.println("Elapsed time : " + watch);
}
Example #25
| Source File: TsneTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Ignore
@Test
public void testTSNEPerformance() throws Exception {
for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {
//STEP 1: Initialization
int iterations = 50;
//create an n-dimensional array of doubles
Nd4j.setDataType(DataType.DOUBLE);
List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words
//STEP 2: Turn text input into a list of words
INDArray weights = Nd4j.rand(10000,300);
StopWatch watch = new StopWatch();
watch.start();
//STEP 3: build a dual-tree tsne to use later
log.info("Build model....");
BarnesHutTsne tsne = new BarnesHutTsne.Builder()
.setMaxIter(iterations)
.theta(0.5)
.normalize(false)
.learningRate(500)
.useAdaGrad(false)
.workspaceMode(wsm)
.build();
watch.stop();
System.out.println("Elapsed time for construction: " + watch);
//STEP 4: establish the tsne values and save them to a file
log.info("Store TSNE Coordinates for Plotting....");
File outDir = testDir.newFolder();
watch.reset();
watch.start();
tsne.fit(weights);
watch.stop();
System.out.println("Elapsed time for fit: " + watch);
tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
}
}
Example #26
| Source File: RandomTests.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
/**
* In this test we check for equality of model params after initialization in different threads
*
* @throws Exception
*/
@Test
public void testModelInitialParamsEquality1() throws Exception {
final List<Model> models = new CopyOnWriteArrayList<>();
for (int i = 0; i < 4; i++) {
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(119) // Training iterations as above
.l2(0.0005)
//.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
.weightInit(WeightInit.XAVIER)
.updater(new Nesterovs(0.01, 0.9))
.trainingWorkspaceMode(WorkspaceMode.ENABLED).list()
.layer(0, new ConvolutionLayer.Builder(5, 5)
//nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
.nIn(1).stride(1, 1).nOut(20).activation(Activation.IDENTITY)
.build())
.layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
.kernelSize(2, 2).stride(2, 2).build())
.layer(2, new ConvolutionLayer.Builder(5, 5)
//Note that nIn need not be specified in later layers
.stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
.layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
.kernelSize(2, 2).stride(2, 2).build())
.layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
.layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
.nOut(10).activation(Activation.SOFTMAX).build())
.setInputType(InputType.convolutionalFlat(28, 28, 1)) //See note below
.build();
MultiLayerNetwork network = new MultiLayerNetwork(conf);
network.init();
models.add(network);
}
});
thread.start();
thread.join();
}
// at the end of day, model params has to
for (int i = 0; i < models.size(); i++) {
assertEquals(models.get(0).params(), models.get(i).params());
}
}
Example #27
| Source File: ValidateMKLDNN.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void compareBatchNormBackward() throws Exception {
assumeTrue(Nd4j.getBackend().getClass().getName().toLowerCase().contains("native"));
Nd4j.getRandom().setSeed(12345);
INDArray in = Nd4j.rand(DataType.FLOAT, 1, 3, 15, 15);
INDArray mean = in.mean(0, 2, 3).reshape(1,3);
INDArray var = in.var(0, 2, 3).reshape(1,3);
INDArray eps = Nd4j.rand(DataType.FLOAT, in.shape());
INDArray gamma = Nd4j.rand(DataType.FLOAT, 1,3);
INDArray beta = Nd4j.rand(DataType.FLOAT, 1,3);
double e = 1e-3;
INDArray dLdIn = in.ulike();
INDArray dLdm = mean.ulike();
INDArray dLdv = var.ulike();
INDArray dLdg = gamma.ulike();
INDArray dLdb = beta.ulike();
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.inferenceWorkspaceMode(WorkspaceMode.NONE)
.trainingWorkspaceMode(WorkspaceMode.NONE)
.list()
.layer(new BatchNormalization.Builder().nIn(3).nOut(3).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
org.deeplearning4j.nn.layers.normalization.BatchNormalization bn = (org.deeplearning4j.nn.layers.normalization.BatchNormalization) net.getLayer(0);
assertNotNull(bn.getHelper());
System.out.println(bn.getHelper());
net.output(in, true);
bn.setInput(in, LayerWorkspaceMgr.noWorkspaces());
Pair<Gradient,INDArray> pcudnn = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces());
Field f = bn.getClass().getDeclaredField("helper");
f.setAccessible(true);
f.set(bn, null);
assertNull(bn.getHelper());
net.output(in, true);
bn.setInput(in, LayerWorkspaceMgr.noWorkspaces());
Pair<Gradient,INDArray> p = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces());
INDArray dldin_dl4j = p.getSecond();
INDArray dldin_helper = pcudnn.getSecond();
assertTrue(dldin_dl4j.equalsWithEps(dldin_helper, 1e-5));
}
Example #28
| Source File: TestTimeDistributed.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testTimeDistributed(){
for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
.trainingWorkspaceMode(wsm)
.inferenceWorkspaceMode(wsm)
.seed(12345)
.updater(new Adam(0.1))
.list()
.layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
.layer(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build())
.layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
.lossFunction(LossFunctions.LossFunction.MCXENT).build())
.setInputType(InputType.recurrent(3, rnnDataFormat))
.build();
MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
.trainingWorkspaceMode(wsm)
.inferenceWorkspaceMode(wsm)
.seed(12345)
.updater(new Adam(0.1))
.list()
.layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
.layer(new TimeDistributed(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build(), rnnDataFormat))
.layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
.lossFunction(LossFunctions.LossFunction.MCXENT).build())
.setInputType(InputType.recurrent(3, rnnDataFormat))
.build();
MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
net1.init();
net2.init();
for( int mb : new int[]{1, 5}) {
for(char inLabelOrder : new char[]{'c', 'f'}) {
INDArray in = Nd4j.rand(DataType.FLOAT, mb, 3, 5).dup(inLabelOrder);
if (rnnDataFormat == RNNFormat.NWC){
in = in.permute(0, 2, 1);
}
INDArray out1 = net1.output(in);
INDArray out2 = net2.output(in);
assertEquals(out1, out2);
INDArray labels ;
if (rnnDataFormat == RNNFormat.NCW) {
labels = TestUtils.randomOneHotTimeSeries(mb, 3, 5).dup(inLabelOrder);
}else{
labels = TestUtils.randomOneHotTimeSeries(mb, 5, 3).dup(inLabelOrder);
}
DataSet ds = new DataSet(in, labels);
net1.fit(ds);
net2.fit(ds);
assertEquals(net1.params(), net2.params());
MultiLayerNetwork net3 = TestUtils.testModelSerialization(net2);
out2 = net2.output(in);
INDArray out3 = net3.output(in);
assertEquals(out2, out3);
}
}
}
}
Example #29
| Source File: TestSameDiffDense.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testSameDiffDenseForward() {
for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
for (int minibatch : new int[]{5, 1}) {
int nIn = 3;
int nOut = 4;
Activation[] afns = new Activation[]{
Activation.TANH,
Activation.SIGMOID,
Activation.ELU,
Activation.IDENTITY,
Activation.SOFTPLUS,
Activation.SOFTSIGN,
Activation.CUBE,
Activation.HARDTANH,
Activation.RELU
};
for (Activation a : afns) {
log.info("Starting test - " + a + ", workspace = " + wsm);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.inferenceWorkspaceMode(wsm)
.trainingWorkspaceMode(wsm)
.list()
.layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
.activation(a)
.build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
assertNotNull(net.paramTable());
MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
.list()
.layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
.build();
MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
net2.init();
net.params().assign(net2.params());
//Check params:
assertEquals(net2.params(), net.params());
Map<String, INDArray> params1 = net.paramTable();
Map<String, INDArray> params2 = net2.paramTable();
assertEquals(params2, params1);
INDArray in = Nd4j.rand(minibatch, nIn);
INDArray out = net.output(in);
INDArray outExp = net2.output(in);
assertEquals(outExp, out);
//Also check serialization:
MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
INDArray outLoaded = netLoaded.output(in);
assertEquals(outExp, outLoaded);
//Sanity check on different minibatch sizes:
INDArray newIn = Nd4j.vstack(in, in);
INDArray outMbsd = net.output(newIn);
INDArray outMb = net2.output(newIn);
assertEquals(outMb, outMbsd);
}
}
}
}
Example #30
| Source File: TestSameDiffDense.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testSameDiffDenseForwardMultiLayer() {
for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
for (int minibatch : new int[]{5, 1}) {
int nIn = 3;
int nOut = 4;
Activation[] afns = new Activation[]{
Activation.TANH,
Activation.SIGMOID,
Activation.ELU,
Activation.IDENTITY,
Activation.SOFTPLUS,
Activation.SOFTSIGN,
Activation.CUBE, //https://github.com/deeplearning4j/nd4j/issues/2426
Activation.HARDTANH,
Activation.RELU //JVM crash
};
for (Activation a : afns) {
log.info("Starting test - " + a + " - workspace=" + wsm);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.seed(12345)
.list()
.layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
.weightInit(WeightInit.XAVIER)
.activation(a).build())
.layer(new SameDiffDense.Builder().nIn(nOut).nOut(nOut)
.weightInit(WeightInit.XAVIER)
.activation(a).build())
.layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
.weightInit(WeightInit.XAVIER)
.activation(a).build())
.validateOutputLayerConfig(false)
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
assertNotNull(net.paramTable());
MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
.seed(12345)
.weightInit(WeightInit.XAVIER)
.list()
.layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
.layer(new DenseLayer.Builder().activation(a).nIn(nOut).nOut(nOut).build())
.layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
.activation(a).build())
.validateOutputLayerConfig(false)
.build();
MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
net2.init();
assertEquals(net2.params(), net.params());
//Check params:
assertEquals(net2.params(), net.params());
Map<String, INDArray> params1 = net.paramTable();
Map<String, INDArray> params2 = net2.paramTable();
assertEquals(params2, params1);
INDArray in = Nd4j.rand(minibatch, nIn);
INDArray out = net.output(in);
INDArray outExp = net2.output(in);
assertEquals(outExp, out);
//Also check serialization:
MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
INDArray outLoaded = netLoaded.output(in);
assertEquals(outExp, outLoaded);
//Sanity check different minibatch sizes
in = Nd4j.rand(2 * minibatch, nIn);
out = net.output(in);
outExp = net2.output(in);
assertEquals(outExp, out);
}
}
}
}
