org.deeplearning4j.nn.conf.WorkspaceMode Java Examples

public static PretrainedNNFilter getPretrainedNNFilterByName(final String name, final int layer,
                                                             final long[] shape) {
    // Thanks to a pointless API requirement, the zoo models require int[] shapes
    // while dl4j uses long[] at any other place
    final int[] intShape = Arrays.stream(shape).mapToInt(i -> (int) i).toArray();

    switch (name) {
        case "AlexNet":
            return new PretrainedNNFilter(new AlexNet(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
        case "LeNet":
            return new PretrainedNNFilter(new LeNet(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
        case "VGG19":
            return new PretrainedNNFilter(new VGG19(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
        case "ResNet50":
            return new PretrainedNNFilter(new ResNet50(42, intShape, 10, WeightInit.DISTRIBUTION,
                    new Nesterovs(1e-2, 0.9), CacheMode.NONE, WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer,
                    shape, name);
        default:
            return new PretrainedNNFilter(new VGG16(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, "VGG16");
    }
}
 

@Test
public void testVaeWeightNoise(){

    for(boolean ws : new boolean[]{false, true}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345L)
                .trainingWorkspaceMode(ws ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                .inferenceWorkspaceMode(ws ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                .weightNoise(new WeightNoise(new org.deeplearning4j.nn.conf.distribution.NormalDistribution(0.1, 0.3)))
                .list().layer(0,
                        new VariationalAutoencoder.Builder().nIn(10).nOut(3)
                                .encoderLayerSizes(5).decoderLayerSizes(6)
                                .pzxActivationFunction(Activation.TANH)
                                .reconstructionDistribution(new GaussianReconstructionDistribution())
                                .activation(new ActivationTanH())
                                .build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray arr = Nd4j.rand(3, 10);
        net.pretrainLayer(0, arr);

    }


}
 

public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter,
                     double realMin, double initialMomentum, double finalMomentum, double momentum,
                     int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance,
                     double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener,
                     double minGain,int vpTreeWorkers) {
    this(numDimensions, simiarlityFunction, theta, invert, maxIter, realMin, initialMomentum, finalMomentum,
            momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, learningRate,
            useAdaGrad, perplexity, TrainingListener, minGain, vpTreeWorkers, WorkspaceMode.NONE, null);
}
 

public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter,
                     double realMin, double initialMomentum, double finalMomentum, double momentum,
                     int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance,
                     double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener,
                     double minGain,int vpTreeWorkers, WorkspaceMode workspaceMode, INDArray staticInput) {
    this.maxIter = maxIter;
    this.realMin = realMin;
    this.initialMomentum = initialMomentum;
    this.finalMomentum = finalMomentum;
    this.momentum = momentum;
    this.normalize = normalize;
    this.useAdaGrad = useAdaGrad;
    this.stopLyingIteration = stopLyingIteration;
    this.learningRate = learningRate;
    this.switchMomentumIteration = switchMomentumIteration;
    this.tolerance = tolerance;
    this.perplexity = perplexity;
    this.minGain = minGain;
    this.numDimensions = numDimensions;
    this.simiarlityFunction = simiarlityFunction;
    this.theta = theta;
    this.trainingListener = TrainingListener;
    this.invert = invert;
    this.vpTreeWorkers = vpTreeWorkers;
    this.workspaceMode = workspaceMode;
    if(this.workspaceMode == null)
        this.workspaceMode = WorkspaceMode.NONE;
    initializer = (staticInput != null) ? new Initializer(staticInput) : new Initializer();
}
 

public SymmetricTrainer(@NonNull Model originalModel, String uuid, int threadIdx, @NonNull WorkspaceMode mode,
                @NonNull ParallelWrapper wrapper, boolean useMDS) {
    super();
    this.uuid = uuid + "_thread_" + threadIdx;
    this.useMDS = useMDS;
    this.originalModel = originalModel;
    this.threadId = threadIdx;
    this.workspaceMode = mode;
    this.parallelWrapper = wrapper;
    this.accumulator = wrapper.getGradientsAccumulator();

}
 

/**
 * Create a {@link Trainer}
 * based on the given parameters
 *
 * @param threadId   the thread id to use for this worker
 * @param model      the model to start the trainer with
 * @param rootDevice the root device id
 * @param useMDS     whether to use MultiDataSet or DataSet
 *                   or not
 * @param wrapper    the wrapper instance to use with this trainer (this refernece is needed
 *                   for coordination with the {@link ParallelWrapper} 's {@link TrainingListener}
 * @return the created training instance
 */
@Override
public Trainer create(String uuid, int threadId, Model model, int rootDevice, boolean useMDS, ParallelWrapper wrapper,
                WorkspaceMode mode, int averagingFrequency) {

    DefaultTrainer trainer = DefaultTrainer.builder().originalModel(model).replicatedModel(model).threadId(threadId)
                    .parallelWrapper(wrapper).workspaceMode(mode).useMDS(useMDS)
                    .uuid(uuid + "_thread_" + threadId)
                    .averagingFrequency(averagingFrequency).build();

    trainer.setName("DefaultTrainer thread " + threadId);
    trainer.setDaemon(true);

    return trainer;
}
 

@Test
public void testSameDiffLamdaVertexBasic(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        log.info("--- Workspace Mode: {} ---", wsm);

        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("lambda", new SameDiffSimpleLambdaVertex(), "0", "1")
                .addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lambda")
                .setOutputs("2")
                .build();

        //Equavalent, not using SameDiff Lambda:
        ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("elementwise", new ElementWiseVertex(ElementWiseVertex.Op.Product), "0", "1")
                .addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "elementwise")
                .setOutputs("3")
                .build();

        ComputationGraph lambda = new ComputationGraph(conf);
        lambda.init();

        ComputationGraph std = new ComputationGraph(confStd);
        std.init();

        lambda.setParams(std.params());

        INDArray in1 = Nd4j.rand(3, 5);
        INDArray in2 = Nd4j.rand(3, 5);
        INDArray labels = TestUtils.randomOneHot(3, 5);
        MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{in1, in2}, new INDArray[]{labels});

        INDArray outLambda = lambda.output(in1, in2)[0];
        INDArray outStd = std.output(in1, in2)[0];

        assertEquals(outLambda, outStd);

        double scoreLambda = lambda.score(mds);
        double scoreStd = std.score(mds);

        assertEquals(scoreStd, scoreLambda, 1e-6);

        for (int i = 0; i < 3; i++) {
            lambda.fit(mds);
            std.fit(mds);

            String s = String.valueOf(i);
            assertEquals(s, std.params(), lambda.params());
            assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
        }

        ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
        outLambda = loaded.output(in1, in2)[0];
        outStd = std.output(in1, in2)[0];

        assertEquals(outStd, outLambda);

        //Sanity check on different minibatch sizes:
        INDArray newIn1 = Nd4j.vstack(in1, in1);
        INDArray newIn2 = Nd4j.vstack(in2, in2);
        INDArray outMbsd = lambda.output(newIn1, newIn2)[0];
        INDArray outMb = std.output(newIn1, newIn2)[0];
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void testROCScoreFunctions() throws Exception {


    for (boolean auc : new boolean[]{true, false}) {
        for (ROCScoreFunction.ROCType rocType : ROCScoreFunction.ROCType.values()) {
            String msg = (auc ? "AUC" : "AUPRC") + " - " + rocType;
            log.info("Starting: " + msg);

            ParameterSpace<Double> lr = new ContinuousParameterSpace(1e-5, 1e-3);

            int nOut = (rocType == ROCScoreFunction.ROCType.ROC ? 2 : 10);
            LossFunctions.LossFunction lf = (rocType == ROCScoreFunction.ROCType.BINARY ?
                    LossFunctions.LossFunction.XENT : LossFunctions.LossFunction.MCXENT);
            Activation a = (rocType == ROCScoreFunction.ROCType.BINARY ? Activation.SIGMOID : Activation.SOFTMAX);
            MultiLayerSpace mls = new MultiLayerSpace.Builder()
                    .trainingWorkspaceMode(WorkspaceMode.NONE)
                    .inferenceWorkspaceMode(WorkspaceMode.NONE)
                    .updater(new AdamSpace(lr))
                    .weightInit(WeightInit.XAVIER)
                    .layer(new OutputLayerSpace.Builder().nIn(784).nOut(nOut)
                            .activation(a)
                            .lossFunction(lf).build())
                    .build();

            CandidateGenerator cg = new RandomSearchGenerator(mls);
            ResultSaver rs = new InMemoryResultSaver();
            ScoreFunction sf = new ROCScoreFunction(rocType, (auc ? ROCScoreFunction.Metric.AUC : ROCScoreFunction.Metric.AUPRC));


            OptimizationConfiguration oc = new OptimizationConfiguration.Builder()
                    .candidateGenerator(cg)
                    .dataProvider(new DP(rocType))
                    .modelSaver(rs)
                    .scoreFunction(sf)
                    .terminationConditions(new MaxCandidatesCondition(3))
                    .rngSeed(12345)
                    .build();

            IOptimizationRunner runner = new LocalOptimizationRunner(oc, new MultiLayerNetworkTaskCreator());
            runner.execute();

            List<ResultReference> list = runner.getResults();

            for (ResultReference rr : list) {
                DataSetIterator testIter = new MnistDataSetIterator(4, 16, false, false, false, 12345);
                testIter.setPreProcessor(new PreProc(rocType));

                OptimizationResult or = rr.getResult();
                MultiLayerNetwork net = (MultiLayerNetwork) or.getResultReference().getResultModel();

                double expScore;
                switch (rocType){
                    case ROC:
                        if(auc){
                            expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUC();
                        } else {
                            expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUCPR();
                        }
                        break;
                    case BINARY:
                        if(auc){
                            expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAuc();
                        } else {
                            expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAUCPR();
                        }
                        break;
                    case MULTICLASS:
                        if(auc){
                            expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUC();
                        } else {
                            expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUCPR();
                        }
                        break;
                    default:
                        throw new RuntimeException();
                }


                DataSetIterator iter = new MnistDataSetIterator(4, 16, false, false, false, 12345);
                iter.setPreProcessor(new PreProc(rocType));

                assertEquals(msg, expScore, or.getScore(), 1e-4);
            }
        }
    }
}
 

@Test
public void testLastTimeStepWithTransfer(){
    int lstmLayerSize = 16;
    int numLabelClasses = 10;
    int numInputs = 5;

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .trainingWorkspaceMode(WorkspaceMode.NONE)
            .inferenceWorkspaceMode(WorkspaceMode.NONE)
            .seed(123)    //Random number generator seed for improved repeatability. Optional.
            .updater(new AdaDelta())
            .weightInit(WeightInit.XAVIER)
            .graphBuilder()
            .addInputs("rr")
            .setInputTypes(InputType.recurrent(30))
            .addLayer("1", new GravesLSTM.Builder().activation(Activation.TANH).nIn(numInputs).nOut(lstmLayerSize).dropOut(0.9).build(), "rr")
            .addLayer("2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(numLabelClasses).build(), "1")

            .setOutputs("2")
            .build();


    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    ComputationGraph updatedModel = new TransferLearning.GraphBuilder(net)
            .addVertex("laststepoutput", new LastTimeStepVertex("rr"), "2")
            .setOutputs("laststepoutput")
            .build();


    INDArray input = Nd4j.rand(new int[]{10, numInputs, 16});

    INDArray[] out = updatedModel.output(input);

    assertNotNull(out);
    assertEquals(1, out.length);
    assertNotNull(out[0]);

    assertArrayEquals(new long[]{10, numLabelClasses}, out[0].shape());

    Map<String,INDArray> acts = updatedModel.feedForward(input, false);

    assertEquals(4, acts.size());   //2 layers + input + vertex output
    assertNotNull(acts.get("laststepoutput"));
    assertArrayEquals(new long[]{10, numLabelClasses}, acts.get("laststepoutput").shape());

    String toString = out[0].toString();
}
 

@Test
public void compareImplementations() throws Exception {

    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .updater(new Adam(1e-3))
                .weightInit(WeightInit.XAVIER)
                .inferenceWorkspaceMode(wsm)
                .trainingWorkspaceMode(wsm)
                .list()
                .layer(new VariationalAutoencoder.Builder()
                        .nIn(784)
                        .nOut(32)
                        .encoderLayerSizes(16)
                        .decoderLayerSizes(16)
                        .activation(Activation.TANH)
                        .pzxActivationFunction(Activation.SIGMOID)
                        .reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID))
                        .build())
                .layer(new VariationalAutoencoder.Builder()
                        .nIn(32)
                        .nOut(8)
                        .encoderLayerSizes(16)
                        .decoderLayerSizes(16)
                        .activation(Activation.TANH)
                        .pzxActivationFunction(Activation.SIGMOID)
                        .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.TANH))
                        .build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf2);
        net.init();

        ComputationGraph cg = net.toComputationGraph();
        cg.getConfiguration().setInferenceWorkspaceMode(wsm);
        cg.getConfiguration().setTrainingWorkspaceMode(wsm);
        DataSetIterator ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
        Nd4j.getRandom().setSeed(12345);
        net.pretrainLayer(0, ds);

        ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
        Nd4j.getRandom().setSeed(12345);
        cg.pretrainLayer("0", ds);

        assertEquals(net.params(), cg.params());
    }
}
 

public static void validateLayers(MultiLayerNetwork net, List<Class<?>> classesToTest, boolean testAllCudnnPresent, int[] fShape, int[] lShape, double maxRE, double minAbsErr) {

        for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

            net.getLayerWiseConfigurations().setTrainingWorkspaceMode(wsm);
            net.getLayerWiseConfigurations().setInferenceWorkspaceMode(wsm);

            Nd4j.getRandom().setSeed(12345);
            INDArray features = Nd4j.rand(fShape);
            INDArray labels = Nd4j.rand(lShape);
            labels = Nd4j.exec(new IsMax(labels, 1))[0].castTo(features.dataType());

            List<CuDNNValidationUtil.TestCase> testCaseList = new ArrayList<>();

            List<DataSet> dataSets = new ArrayList<>();
            for (int i = 0; i < 6; i++) {
                INDArray f = Nd4j.rand(fShape);
                INDArray l = Nd4j.rand(lShape);
                l = Nd4j.exec(new IsMax(l, 1))[0].castTo(features.dataType());
                dataSets.add(new DataSet(f, l));
            }
            DataSetIterator iter = new ExistingDataSetIterator(dataSets);

            for (Class<?> c : classesToTest) {
                String name = "WS=" + wsm + ", testCudnnFor=" + c.getSimpleName();
                testCaseList.add(CuDNNValidationUtil.TestCase.builder()
                        .testName(name)
                        .allowCudnnHelpersForClasses(Collections.<Class<?>>singletonList(c))
                        .testForward(true)
                        .testScore(true)
                        .testBackward(true)
                        .testTraining(true)
                        .trainFirst(false)
                        .features(features)
                        .labels(labels)
                        .data(iter)
                        .maxRE(maxRE)
                        .minAbsErr(minAbsErr)
                        .build());
            }

            if(testAllCudnnPresent) {
                testCaseList.add(CuDNNValidationUtil.TestCase.builder()
                        .testName("WS=" + wsm + ", ALL CLASSES")
                        .allowCudnnHelpersForClasses(classesToTest)
                        .testForward(true)
                        .testScore(true)
                        .testBackward(true)
                        .trainFirst(false)
                        .features(features)
                        .labels(labels)
                        .data(iter)
                        .maxRE(maxRE)
                        .minAbsErr(minAbsErr)
                        .build());
            }

            for (CuDNNValidationUtil.TestCase tc : testCaseList) {
                log.info("Running test: " + tc.getTestName());
                CuDNNValidationUtil.validateMLN(net, tc);
            }
        }
    }
 

@Test
public void testEqualUuid1() {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
            .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(2, new ConvolutionLayer.Builder(5, 5)
                    //Note that nIn needed be specified in later layers
                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
            .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
            // DataSets prefetching options. Set this value with respect to number of actual devices
            .prefetchBuffer(24)

            // set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
            .workers(2)

            // rare averaging improves performance, but might reduce model accuracy
            .averagingFrequency(3)

            // if set to TRUE, on every averaging model score will be reported
            .reportScoreAfterAveraging(true)

            // optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
            .build();

    val trainer = new SymmetricTrainer(model, "alpha", 3, WorkspaceMode.NONE, wrapper, true);

    assertEquals("alpha_thread_3", trainer.getUuid());
}
 

@Test
public void testEqualUuid1() {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
            .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(2, new ConvolutionLayer.Builder(5, 5)
                    //Note that nIn needed be specified in later layers
                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
            .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
            // DataSets prefetching options. Set this value with respect to number of actual devices
            .prefetchBuffer(24)

            // set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
            .workers(2)

            // rare averaging improves performance, but might reduce model accuracy
            .averagingFrequency(3)

            // if set to TRUE, on every averaging model score will be reported
            .reportScoreAfterAveraging(true)

            // optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
            .build();

    val context = new DefaultTrainerContext();
    val trainer = context.create("alpha", 3, model, 0, true, wrapper, WorkspaceMode.NONE, 3);

    assertEquals("alpha_thread_3", trainer.getUuid());
}
 

public Builder trainingWorkspaceMode(WorkspaceMode workspaceMode){
    this.trainingWorkspaceMode = workspaceMode;
    return this;
}
 

public Builder inferenceWorkspaceMode(WorkspaceMode workspaceMode){
    this.inferenceWorkspaceMode = workspaceMode;
    return this;
}
 

public Builder trainingWorkspaceMode(WorkspaceMode workspaceMode){
    this.trainingWorkspaceMode = workspaceMode;
    return this;
}
 

public Builder inferenceWorkspaceMode(WorkspaceMode workspaceMode){
    this.inferenceWorkspaceMode = workspaceMode;
    return this;
}
 

public ComputationGraph init() {
    	ComputationGraphConfiguration.GraphBuilder graph = new NeuralNetConfiguration.Builder()
                .seed(seed)
                .iterations(iterations)
                .activation(Activation.LEAKYRELU)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                .lrPolicyDecayRate(0.5)
                .learningRateDecayPolicy(LearningRatePolicy.Score)
                .updater(Adam.builder().build())
                .weightInit(WeightInit.XAVIER)
                .learningRate(0.02)
                .miniBatch(miniBatch)
                .convolutionMode(ConvolutionMode.Truncate)
                .trainingWorkspaceMode(WorkspaceMode.SINGLE)
                .inferenceWorkspaceMode(WorkspaceMode.SINGLE)
                .graphBuilder();
    	
    	// set input & output
		graph
			.addInputs("input").setInputTypes(InputType.convolutionalFlat(height, width, channels))
			.addLayer("policy", new OutputLayer.Builder()
	                .lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
	                .activation(Activation.SOFTMAX)
	                .nOut(numClasses).build(), "embeddings_c")
			.addLayer("value1", new OutputLayer.Builder()
	                .lossFunction(LossFunctions.LossFunction.MSE)
	                .activation(Activation.TANH)
	                .nOut(1).build(), "embeddings_r1")
//	        .addLayer("value2", new OutputLayer.Builder()
//	                .lossFunction(LossFunctions.LossFunction.MSE)
//	                .activation(Activation.TANH)
//	                .nOut(1).build(), "embeddings_r2")
	        .setOutputs("policy", "value1", "value2")
			.backprop(true).pretrain(false);
		
		int kernelSize = 128;
		
		graph.addLayer("c-layer0", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.LEAKYRELU).nOut(kernelSize).build(), "input");
		
		int blockNum = 8;
		String prevLayer = "c-layer0";
		for (int i = 1; i <= blockNum; i++) {
			String layerName = "c-block" + i + "-";
			graph.addLayer(layerName + "1", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.LEAKYRELU).nOut(kernelSize).build(), prevLayer);
			graph.addLayer(layerName + "2", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(kernelSize).build(), layerName + "1");
			graph.addVertex("shortcut" + i, new ElementWiseVertex(ElementWiseVertex.Op.Add), layerName + "2", prevLayer);
			graph.addLayer(layerName + "3", new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build(), "shortcut" + i);
			prevLayer = layerName + "3";
		}
		
		// for classification
		graph.addLayer("embeddings_c", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(2).build(), prevLayer);
		
		// for value regression
		graph.addLayer("reg-c-layer1", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(1).build(), prevLayer);
		graph.addLayer("embeddings_r1", new DenseLayer.Builder().activation(Activation.IDENTITY).nOut(256).build(), "reg-c-layer1");
		
//		graph.addLayer("reg-c-layer2", new ConvolutionLayer.Builder(new int[]{3,3}, new int[]{1,1}, new int[]{1,1}).activation(Activation.IDENTITY).nOut(1).build(), prevLayer);
//		graph.addLayer("embeddings_r2", new DenseLayer.Builder().activation(Activation.IDENTITY).nOut(256).build(), "reg-c-layer2");

		ComputationGraphConfiguration conf = graph.build();
		ComputationGraph model = new ComputationGraph(conf);
		model.init();
		
		log.info("\nNumber of params: " + model.numParams()+"\n");
		return model;
    }
 

@Test
public void testSameDiffConvGradient() {
    int imgH = 8;
    int imgW = 8;
    int nIn = 3;
    int nOut = 4;
    int[] kernel = {2, 2};
    int[] strides = {1, 1};
    int[] dilation = {1, 1};

    int count = 0;

    //Note: to avoid the exporential number of tests here, we'll randomly run every Nth test only.
    //With n=1, m=3 this is 1 out of every 3 tests (on average)
    Random r = new Random(12345);
    int n = 1;
    int m = 5;
    for(boolean workspaces : new boolean[]{false, true}) {
        for (int minibatch : new int[]{5, 1}) {
            for (boolean hasBias : new boolean[]{true, false}) {
                for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) {
                    int i = r.nextInt(m);
                    if (i >= n) {
                        //Example: n=2, m=3... skip on i=2, run test on i=0, i=1
                        continue;
                    }

                    String msg = "Test " + (count++) + " - minibatch=" + minibatch + ", ConvolutionMode=" + cm + ", hasBias=" + hasBias;

                    int outH = cm == ConvolutionMode.Same ? imgH : (imgH-2);
                    int outW = cm == ConvolutionMode.Same ? imgW : (imgW-2);

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .dataType(DataType.DOUBLE)
                            .seed(12345)
                            .updater(new NoOp())
                            .trainingWorkspaceMode(workspaces ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                            .inferenceWorkspaceMode(workspaces ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                            .list()
                            .layer(new SameDiffConv.Builder()
                                    .weightInit(WeightInit.XAVIER)
                                    .nIn(nIn)
                                    .nOut(nOut)
                                    .kernelSize(kernel)
                                    .stride(strides)
                                    .dilation(dilation)
                                    .convolutionMode(cm)
                                    .activation(Activation.TANH)
                                    .hasBias(hasBias)
                                    .build())
                            .layer(new SameDiffConv.Builder()
                                    .weightInit(WeightInit.XAVIER)
                                    .nIn(nOut)
                                    .nOut(nOut)
                                    .kernelSize(kernel)
                                    .stride(strides)
                                    .dilation(dilation)
                                    .convolutionMode(cm)
                                    .activation(Activation.SIGMOID)
                                    .hasBias(hasBias)
                                    .build())
                            .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .nIn(nOut * outH * outW)
                                    .nOut(nOut).build())
                            .inputPreProcessor(2, new CnnToFeedForwardPreProcessor(outH, outW, nOut))
                            .build();

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    INDArray f = Nd4j.rand(new int[]{minibatch, nIn, imgH, imgW});
                    INDArray l = TestUtils.randomOneHot(minibatch, nOut);

                    log.info("Starting: " + msg);
                    boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(f)
                            .labels(l).subset(true).maxPerParam(50));

                    assertTrue(msg, gradOK);

                    TestUtils.testModelSerialization(net);

                    //Sanity check on different minibatch sizes:
                    INDArray newIn = Nd4j.vstack(f, f);
                    net.output(newIn);
                }
            }
        }
    }
}
 

private void train() throws IOException {

        ZooModel zooModel = VGG16.builder().build();
        ComputationGraph vgg16 = (ComputationGraph) zooModel.initPretrained(PretrainedType.CIFAR10);
        log.info(vgg16.summary());

        IUpdater iUpdaterWithDefaultConfig = Updater.ADAM.getIUpdaterWithDefaultConfig();
        iUpdaterWithDefaultConfig.setLrAndSchedule(0.1, null);
        FineTuneConfiguration fineTuneConf = new FineTuneConfiguration.Builder()
                .seed(1234)
//                .weightInit(WeightInit.XAVIER)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .activation(Activation.RELU)
                .updater(iUpdaterWithDefaultConfig)
                .cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
                .miniBatch(true)
                .inferenceWorkspaceMode(WorkspaceMode.ENABLED)
                .trainingWorkspaceMode(WorkspaceMode.ENABLED)
                .pretrain(true)
                .backprop(true)
                .build();

        ComputationGraph cifar10 = new TransferLearning.GraphBuilder(vgg16)
                .setWorkspaceMode(WorkspaceMode.ENABLED)
                .fineTuneConfiguration(fineTuneConf)
                .setInputTypes(InputType.convolutionalFlat(ImageUtils.HEIGHT,
                        ImageUtils.WIDTH, 3))
                .removeVertexAndConnections("dense_2_loss")
                .removeVertexAndConnections("dense_2")
                .removeVertexAndConnections("dense_1")
                .removeVertexAndConnections("dropout_1")
                .removeVertexAndConnections("embeddings")
                .removeVertexAndConnections("flatten_1")
                .addLayer("dense_1", new DenseLayer.Builder()
                        .nIn(4096)
                        .nOut(EMBEDDINGS)
                        .activation(Activation.RELU).build(), "block3_pool")
                .addVertex("embeddings", new L2NormalizeVertex(new int[]{}, 1e-12), "dense_1")
                .addLayer("lossLayer", new CenterLossOutputLayer.Builder()
                                .lossFunction(LossFunctions.LossFunction.SQUARED_LOSS)
                                .activation(Activation.SOFTMAX).nIn(EMBEDDINGS).nOut(NUM_POSSIBLE_LABELS)
                                .lambda(LAMBDA).alpha(0.9)
                                .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).build(),
                        "embeddings")
                .setOutputs("lossLayer")
                .build();

        log.info(cifar10.summary());
        File rootDir = new File("CarTracking/train_from_video_" + NUM_POSSIBLE_LABELS);
        DataSetIterator dataSetIterator = ImageUtils.createDataSetIterator(rootDir, NUM_POSSIBLE_LABELS, BATCH_SIZE);
        DataSetIterator testIterator = ImageUtils.createDataSetIterator(rootDir, NUM_POSSIBLE_LABELS, BATCH_SIZE);
        cifar10.setListeners(new ScoreIterationListener(2));
        int iEpoch = I_EPOCH;
        while (iEpoch < EPOCH_TRAINING) {
            while (dataSetIterator.hasNext()) {
                DataSet trainMiniBatchData = null;
                try {
                    trainMiniBatchData = dataSetIterator.next();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                cifar10.fit(trainMiniBatchData);
            }
            iEpoch++;

            String modelName = PREFIX + NUM_POSSIBLE_LABELS + "_epoch_data_e" + EMBEDDINGS + "_b" + BATCH_SIZE + "_" + iEpoch + ".zip";
            saveProgress(cifar10, iEpoch, modelName);
            testResults(cifar10, testIterator, iEpoch, modelName);
            dataSetIterator.reset();
            log.info("iEpoch = " + iEpoch);
        }
    }
 

private ComputationGraph VasttextTextualAndNumeric()
{
	Activation activation = null;
	LossFunction loss = null;
	//If multilabel, it is considered according to the book "Deep Learning with Python" to use the following parameters
	if(multiLabel)
	{
		activation = Activation.SIGMOID;
		loss = LossFunction.XENT; //Binary Crossentropy
	}
	else
	{
		//We're using a softmax/cross entropy for the binary classification, as the number of neurons is two. If the number of neurons would be one, then
		//the activation would be sigmoid and the loss binary crossentropy
		activation = Activation.SOFTMAX;
		loss = LossFunction.MCXENT;	//CATEGORICAL_CROSSENTROPY
	}

	System.err.println("LR:"+lr);
	
	System.err.println("Dense:"+denseDimension);

	ComputationGraphConfiguration  nnConf = new NeuralNetConfiguration.Builder()
			.updater(new Adam(lr))
			.weightInit(WeightInit.XAVIER)
			.trainingWorkspaceMode(WorkspaceMode.ENABLED)
               .inferenceWorkspaceMode(WorkspaceMode.ENABLED)
			.graphBuilder()
			.addInputs("Text", "Extra")
			//Embeddings Parts
			.addLayer("Embeddings", new EmbeddingSequenceLayer.Builder()
                       .nIn(textFeaturesSize)
                       .nOut(denseDimension)
                       .activation(Activation.IDENTITY)
                       //.activation(Activation.TANH)
                       //.dropOut(0.0)
                       .build(), "Text")
			.addLayer("GlobalPooling", new GlobalPoolingLayer.Builder()
                       .poolingType(PoolingType.AVG)
                       .poolingDimensions(2)
                       .collapseDimensions(true)
                       //.dropOut(0.0)
                       .build(), "Embeddings")
			//We're merging directly the values from the extra
			.addVertex("Merge", new MergeVertex(), "GlobalPooling","Extra")
			.addLayer("DenseAll", new DenseLayer.Builder()
					.nIn(denseDimension+numericFeaturesSize)
					.nOut(denseDimension/2)
					//.dropOut(0.5)
					//.l2(0.001)
					.build(), "Merge")
			.addLayer("Output", new OutputLayer.Builder()
					//.dropOut(0.5)
					.nIn(denseDimension/2)
                       .nOut(labelsSize)
                       .activation(activation)
                       .lossFunction(loss)
                       .build(), "DenseAll")
			.setOutputs("Output")
			.pretrain(false)
			.backprop(true)
			.build();

	return new ComputationGraph(nnConf);
}
 

private MultiLayerNetwork VasttextTextual()
{
	Activation activation = null;
	LossFunction loss = null;
	//If multilabel, it is considered according to the book "Deep Learning with Python" to use the following parameters
	if(multiLabel)
	{
		activation = Activation.SIGMOID;
		loss = LossFunction.XENT; //Binary Crossentropy
	}
	else
	{
		//We're using a softmax/cross entropy for the binary classification, as the number of neurons is two. If the number of neurons would be one, then
		//the activation would be sigmoid and the loss binary crossentropy
		activation = Activation.SOFTMAX;
		loss = LossFunction.MCXENT;	//CATEGORICAL_CROSSENTROPY
	}

	MultiLayerConfiguration nnConf = new NeuralNetConfiguration.Builder()
               .updater(new Adam(lr))
               .weightInit(WeightInit.XAVIER)
               .trainingWorkspaceMode(WorkspaceMode.ENABLED)
               .inferenceWorkspaceMode(WorkspaceMode.ENABLED)
               .list()
               .layer(0, new EmbeddingSequenceLayer.Builder()
                       .nIn(textFeaturesSize)
                       .nOut(denseDimension)
                       .activation(Activation.IDENTITY)
                       .build())
               .layer(1, new GlobalPoolingLayer.Builder()
                       .poolingType(PoolingType.AVG)
                       .poolingDimensions(2)
                       .collapseDimensions(true)
                       .build())
               .layer(2, new OutputLayer.Builder()
                       .nIn(denseDimension)
                       .nOut(labelsSize)
                       .activation(activation)
                       .lossFunction(loss)
                       .build())
               .pretrain(false).backprop(true).build();

       return new MultiLayerNetwork(nnConf);
}
 

@Test
public void testSimple() throws Exception {
    //Simple sanity check

    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(250, 200);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
}
 

@Test
public void testPerformance() throws Exception {

    StopWatch watch = new StopWatch();
    watch.start();
    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(DataType.FLOAT, 250, 20);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
    watch.stop();
    System.out.println("Elapsed time : " + watch);
}
 

@Ignore
@Test
public void testTSNEPerformance() throws Exception {

        for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

            //STEP 1: Initialization
            int iterations = 50;
            //create an n-dimensional array of doubles
            Nd4j.setDataType(DataType.DOUBLE);
            List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

            //STEP 2: Turn text input into a list of words
            INDArray weights = Nd4j.rand(10000,300);

            StopWatch watch = new StopWatch();
            watch.start();
            //STEP 3: build a dual-tree tsne to use later
            log.info("Build model....");
            BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                    .setMaxIter(iterations)
                    .theta(0.5)
                    .normalize(false)
                    .learningRate(500)
                    .useAdaGrad(false)
                    .workspaceMode(wsm)
                    .build();

            watch.stop();
            System.out.println("Elapsed time for construction: " + watch);

            //STEP 4: establish the tsne values and save them to a file
            log.info("Store TSNE Coordinates for Plotting....");
            File outDir = testDir.newFolder();

            watch.reset();
            watch.start();
            tsne.fit(weights);
            watch.stop();
            System.out.println("Elapsed time for fit: " + watch);
            tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
        }
}
 

/**
 * In this test we check for equality of model params after initialization in different threads
 *
 * @throws Exception
 */
@Test
public void testModelInitialParamsEquality1() throws Exception {
    final List<Model> models = new CopyOnWriteArrayList<>();

    for (int i = 0; i < 4; i++) {
        Thread thread = new Thread(new Runnable() {
            @Override
            public void run() {
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(119) // Training iterations as above
                                .l2(0.0005)
                                //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
                                .weightInit(WeightInit.XAVIER)
                                .updater(new Nesterovs(0.01, 0.9))
                                .trainingWorkspaceMode(WorkspaceMode.ENABLED).list()
                                .layer(0, new ConvolutionLayer.Builder(5, 5)
                                                //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                                                .nIn(1).stride(1, 1).nOut(20).activation(Activation.IDENTITY)
                                                .build())
                                .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                                                .kernelSize(2, 2).stride(2, 2).build())
                                .layer(2, new ConvolutionLayer.Builder(5, 5)
                                                //Note that nIn need not be specified in later layers
                                                .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
                                .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                                                .kernelSize(2, 2).stride(2, 2).build())
                                .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
                                .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                                .nOut(10).activation(Activation.SOFTMAX).build())
                                .setInputType(InputType.convolutionalFlat(28, 28, 1)) //See note below
                                .build();

                MultiLayerNetwork network = new MultiLayerNetwork(conf);
                network.init();

                models.add(network);
            }
        });

        thread.start();
        thread.join();
    }


    // at the end of day, model params has to
    for (int i = 0; i < models.size(); i++) {
        assertEquals(models.get(0).params(), models.get(i).params());
    }
}
 

@Test
public void compareBatchNormBackward() throws Exception {
    assumeTrue(Nd4j.getBackend().getClass().getName().toLowerCase().contains("native"));

    Nd4j.getRandom().setSeed(12345);
    INDArray in = Nd4j.rand(DataType.FLOAT, 1, 3, 15, 15);
    INDArray mean = in.mean(0, 2, 3).reshape(1,3);
    INDArray var = in.var(0, 2, 3).reshape(1,3);
    INDArray eps = Nd4j.rand(DataType.FLOAT, in.shape());
    INDArray gamma = Nd4j.rand(DataType.FLOAT, 1,3);
    INDArray beta = Nd4j.rand(DataType.FLOAT, 1,3);
    double e = 1e-3;

    INDArray dLdIn = in.ulike();
    INDArray dLdm = mean.ulike();
    INDArray dLdv = var.ulike();
    INDArray dLdg = gamma.ulike();
    INDArray dLdb = beta.ulike();


    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .inferenceWorkspaceMode(WorkspaceMode.NONE)
            .trainingWorkspaceMode(WorkspaceMode.NONE)
            .list()
            .layer(new BatchNormalization.Builder().nIn(3).nOut(3).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    org.deeplearning4j.nn.layers.normalization.BatchNormalization bn = (org.deeplearning4j.nn.layers.normalization.BatchNormalization) net.getLayer(0);
    assertNotNull(bn.getHelper());
    System.out.println(bn.getHelper());

    net.output(in, true);
    bn.setInput(in, LayerWorkspaceMgr.noWorkspaces());
    Pair<Gradient,INDArray> pcudnn = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces());

    Field f = bn.getClass().getDeclaredField("helper");
    f.setAccessible(true);
    f.set(bn, null);
    assertNull(bn.getHelper());

    net.output(in, true);
    bn.setInput(in, LayerWorkspaceMgr.noWorkspaces());
    Pair<Gradient,INDArray> p = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces());

    INDArray dldin_dl4j = p.getSecond();
    INDArray dldin_helper = pcudnn.getSecond();

    assertTrue(dldin_dl4j.equalsWithEps(dldin_helper, 1e-5));
}
 

@Test
public void testTimeDistributed(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {

        MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
                .layer(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build())
                .layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(3, rnnDataFormat))
                .build();

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
                .layer(new TimeDistributed(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build(), rnnDataFormat))
                .layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(3, rnnDataFormat))
                .build();

        MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net1.init();
        net2.init();

        for( int mb : new int[]{1, 5}) {
            for(char inLabelOrder : new char[]{'c', 'f'}) {
                INDArray in = Nd4j.rand(DataType.FLOAT, mb, 3, 5).dup(inLabelOrder);
                if (rnnDataFormat == RNNFormat.NWC){
                    in = in.permute(0, 2, 1);
                }
                INDArray out1 = net1.output(in);
                INDArray out2 = net2.output(in);
                assertEquals(out1, out2);

                INDArray labels ;
                if (rnnDataFormat == RNNFormat.NCW) {
                    labels = TestUtils.randomOneHotTimeSeries(mb, 3, 5).dup(inLabelOrder);
                }else{
                    labels = TestUtils.randomOneHotTimeSeries(mb, 5, 3).dup(inLabelOrder);
                }



                DataSet ds = new DataSet(in, labels);
                net1.fit(ds);
                net2.fit(ds);

                assertEquals(net1.params(), net2.params());

                MultiLayerNetwork net3 = TestUtils.testModelSerialization(net2);
                out2 = net2.output(in);
                INDArray out3 = net3.output(in);

                assertEquals(out2, out3);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseForward() {
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        for (int minibatch : new int[]{5, 1}) {
            int nIn = 3;
            int nOut = 4;

            Activation[] afns = new Activation[]{
                    Activation.TANH,
                    Activation.SIGMOID,
                    Activation.ELU,
                    Activation.IDENTITY,
                    Activation.SOFTPLUS,
                    Activation.SOFTSIGN,
                    Activation.CUBE,
                    Activation.HARDTANH,
                    Activation.RELU
            };

            for (Activation a : afns) {
                log.info("Starting test - " + a + ", workspace = " + wsm);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .inferenceWorkspaceMode(wsm)
                        .trainingWorkspaceMode(wsm)
                        .list()
                        .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                .activation(a)
                                .build())
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                assertNotNull(net.paramTable());

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .list()
                        .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                        .build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                net.params().assign(net2.params());

                //Check params:
                assertEquals(net2.params(), net.params());
                Map<String, INDArray> params1 = net.paramTable();
                Map<String, INDArray> params2 = net2.paramTable();
                assertEquals(params2, params1);

                INDArray in = Nd4j.rand(minibatch, nIn);
                INDArray out = net.output(in);
                INDArray outExp = net2.output(in);

                assertEquals(outExp, out);

                //Also check serialization:
                MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
                INDArray outLoaded = netLoaded.output(in);

                assertEquals(outExp, outLoaded);

                //Sanity check on different minibatch sizes:
                INDArray newIn = Nd4j.vstack(in, in);
                INDArray outMbsd = net.output(newIn);
                INDArray outMb = net2.output(newIn);
                assertEquals(outMb, outMbsd);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseForwardMultiLayer() {
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        for (int minibatch : new int[]{5, 1}) {
            int nIn = 3;
            int nOut = 4;

            Activation[] afns = new Activation[]{
                    Activation.TANH,
                    Activation.SIGMOID,
                    Activation.ELU,
                    Activation.IDENTITY,
                    Activation.SOFTPLUS,
                    Activation.SOFTSIGN,
                    Activation.CUBE,    //https://github.com/deeplearning4j/nd4j/issues/2426
                    Activation.HARDTANH,
                    Activation.RELU      //JVM crash
            };

            for (Activation a : afns) {
                log.info("Starting test - " + a + " - workspace=" + wsm);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .seed(12345)
                        .list()
                        .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .layer(new SameDiffDense.Builder().nIn(nOut).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .validateOutputLayerConfig(false)
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                assertNotNull(net.paramTable());

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .seed(12345)
                        .weightInit(WeightInit.XAVIER)
                        .list()
                        .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                        .layer(new DenseLayer.Builder().activation(a).nIn(nOut).nOut(nOut).build())
                        .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
                                .activation(a).build())
                        .validateOutputLayerConfig(false)
                        .build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                assertEquals(net2.params(), net.params());

                //Check params:
                assertEquals(net2.params(), net.params());
                Map<String, INDArray> params1 = net.paramTable();
                Map<String, INDArray> params2 = net2.paramTable();
                assertEquals(params2, params1);

                INDArray in = Nd4j.rand(minibatch, nIn);
                INDArray out = net.output(in);
                INDArray outExp = net2.output(in);

                assertEquals(outExp, out);

                //Also check serialization:
                MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
                INDArray outLoaded = netLoaded.output(in);

                assertEquals(outExp, outLoaded);


                //Sanity check different minibatch sizes
                in = Nd4j.rand(2 * minibatch, nIn);
                out = net.output(in);
                outExp = net2.output(in);
                assertEquals(outExp, out);
            }
        }
    }
}