org.deeplearning4j.nn.conf.layers.LossLayer Java Examples

/**
 * Get DL4J LossLayer.
 *
 * @return LossLayer
 */
public FeedForwardLayer getLossLayer(InputType type) throws UnsupportedKerasConfigurationException {
    if (type instanceof InputType.InputTypeFeedForward) {
        this.layer = new LossLayer.Builder(loss).name(this.layerName).activation(Activation.IDENTITY).build();
    }
    else if (type instanceof  InputType.InputTypeRecurrent) {
        this.layer = new RnnLossLayer.Builder(loss).name(this.layerName).activation(Activation.IDENTITY).build();
    }
    else if (type instanceof InputType.InputTypeConvolutional) {
        this.layer = new CnnLossLayer.Builder(loss).name(this.layerName).activation(Activation.IDENTITY).build();
    } else {
        throw new UnsupportedKerasConfigurationException("Unsupported output layer type"
                + "got : " + type.toString());
    }
    return (FeedForwardLayer) this.layer;
}
 

@Test
public void importSparseXent() throws Exception {
    String modelPath = "modelimport/keras/examples/simple_sparse_xent/simple_sparse_xent_mlp_keras_2_model.h5";
    String inputsOutputPath = "modelimport/keras/examples/simple_sparse_xent/simple_sparse_xent_mlp_keras_2_inputs_and_outputs.h5";
    MultiLayerNetwork net = importEndModelTest(modelPath, inputsOutputPath, true, true, true, true);
    Layer outLayer = net.getOutputLayer();
    assertTrue(outLayer instanceof org.deeplearning4j.nn.layers.LossLayer);
    LossLayer llConf = (LossLayer) outLayer.getConfig();
    assertEquals(new LossSparseMCXENT(), llConf.getLossFn());
}
 

@Test
public void testTransferLearning() throws Exception {
    ignoreIfCuda();
    //https://github.com/deeplearning4j/deeplearning4j/issues/7193

    ComputationGraph cg = (ComputationGraph) ResNet50.builder().build().initPretrained();

    cg = new TransferLearning.GraphBuilder(cg)
            .addLayer("out", new LossLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.IDENTITY).build(), "fc1000")
            .setInputTypes(InputType.convolutional(224, 224, 3))
            .setOutputs("out")
            .build();

}
 

@Override
public LossLayer getValue(double[] parameterValues) {
    LossLayer.Builder b = new LossLayer.Builder();
    if(activationFunction != null)
        b.activation(activationFunction.getValue(parameterValues));
    if(lossFunction != null)
        b.lossFunction(lossFunction.getValue(parameterValues));
    return b.build();
}
 

@Test
public void testCapsNet() {

    int[] minibatchSizes = {8, 16};

    int width = 6;
    int height = 6;
    int inputDepth = 4;

    int[] primaryCapsDims = {2, 4};
    int[] primaryCapsChannels = {8};
    int[] capsules = {5};
    int[] capsuleDims = {4, 8};
    int[] routings = {1};

    Nd4j.getRandom().setSeed(12345);

    for (int routing : routings) {
        for (int primaryCapsDim : primaryCapsDims) {
            for (int primarpCapsChannel : primaryCapsChannels) {
                for (int capsule : capsules) {
                    for (int capsuleDim : capsuleDims) {
                        for (int minibatchSize : minibatchSizes) {

                            INDArray input = Nd4j.rand(minibatchSize, inputDepth * height * width).mul(10)
                                    .reshape(-1, inputDepth, height, width);
                            INDArray labels = Nd4j.zeros(minibatchSize, capsule);
                            for (int i = 0; i < minibatchSize; i++) {
                                labels.putScalar(new int[]{i, i % capsule}, 1.0);
                            }

                            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                    .dataType(DataType.DOUBLE)
                                    .seed(123)
                                    .updater(new NoOp())
                                    .weightInit(new WeightInitDistribution(new UniformDistribution(-6, 6)))
                                    .list()
                                    .layer(new PrimaryCapsules.Builder(primaryCapsDim, primarpCapsChannel)
                                            .kernelSize(3, 3)
                                            .stride(2, 2)
                                            .build())
                                    .layer(new CapsuleLayer.Builder(capsule, capsuleDim, routing).build())
                                    .layer(new CapsuleStrengthLayer.Builder().build())
                                    .layer(new ActivationLayer.Builder(new ActivationSoftmax()).build())
                                    .layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build())
                                    .setInputType(InputType.convolutional(height, width, inputDepth))
                                    .build();

                            MultiLayerNetwork net = new MultiLayerNetwork(conf);
                            net.init();

                            for (int i = 0; i < 4; i++) {
                                System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
                            }

                            String msg = "minibatch=" + minibatchSize +
                                    ", PrimaryCaps: " + primarpCapsChannel +
                                    " channels, " + primaryCapsDim + " dimensions, Capsules: " + capsule +
                                    " capsules with " + capsuleDim + " dimensions and " + routing + " routings";
                            System.out.println(msg);

                            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
                                    .labels(labels).subset(true).maxPerParam(100));

                            assertTrue(msg, gradOK);

                            TestUtils.testModelSerialization(net);
                        }
                    }
                }
            }
        }
    }
}
 

@Test
public void testOutputMSELossLayer(){
    Nd4j.getRandom().setSeed(12345);

    MultiLayerConfiguration confSD = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .updater(new Adam(0.01))
            .list()
            .layer(new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build())
            .layer(new SameDiffMSELossLayer())
            .build();

    MultiLayerConfiguration confStd = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .updater(new Adam(0.01))
            .list()
            .layer(new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build())
            .layer(new LossLayer.Builder().activation(Activation.IDENTITY).lossFunction(LossFunctions.LossFunction.MSE).build())
            .build();

    MultiLayerNetwork netSD = new MultiLayerNetwork(confSD);
    netSD.init();

    MultiLayerNetwork netStd = new MultiLayerNetwork(confStd);
    netStd.init();

    INDArray in = Nd4j.rand(3, 5);
    INDArray label = Nd4j.rand(3,5);

    INDArray outSD = netSD.output(in);
    INDArray outStd = netStd.output(in);
    assertEquals(outStd, outSD);

    DataSet ds = new DataSet(in, label);
    double scoreSD = netSD.score(ds);
    double scoreStd = netStd.score(ds);
    assertEquals(scoreStd, scoreSD, 1e-6);

    for( int i=0; i<3; i++ ){
        netSD.fit(ds);
        netStd.fit(ds);

        assertEquals(netStd.params(), netSD.params());
        assertEquals(netStd.getFlattenedGradients(), netSD.getFlattenedGradients());
    }

    //Test fit before output:
    MultiLayerNetwork net = new MultiLayerNetwork(confSD.clone());
    net.init();
    net.fit(ds);

    //Sanity check on different minibatch sizes:
    INDArray newIn = Nd4j.vstack(in, in);
    INDArray outMbsd = netSD.output(newIn);
    INDArray outMb = netStd.output(newIn);
    assertEquals(outMb, outMbsd);
}
 

@Test
public void testCapsNetOnMNIST(){
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(123)
            .updater(new Adam())
            .list()
            .layer(new ConvolutionLayer.Builder()
                    .nOut(16)
                    .kernelSize(9, 9)
                    .stride(3, 3)
                    .build())
            .layer(new PrimaryCapsules.Builder(8, 8)
                    .kernelSize(7, 7)
                    .stride(2, 2)
                    .build())
            .layer(new CapsuleLayer.Builder(10, 16, 3).build())
            .layer(new CapsuleStrengthLayer.Builder().build())
            .layer(new ActivationLayer.Builder(new ActivationSoftmax()).build())
            .layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build())
            .setInputType(InputType.convolutionalFlat(28, 28, 1))
            .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    int rngSeed = 12345;
    try {
        MnistDataSetIterator mnistTrain = new MnistDataSetIterator(64, true, rngSeed);
        MnistDataSetIterator mnistTest = new MnistDataSetIterator(64, false, rngSeed);

        for (int i = 0; i < 2; i++) {
            model.fit(mnistTrain);
        }

        Evaluation eval = model.evaluate(mnistTest);

        assertTrue("Accuracy not over 95%", eval.accuracy() > 0.95);
        assertTrue("Precision not over 95%", eval.precision() > 0.95);
        assertTrue("Recall not over 95%", eval.recall() > 0.95);
        assertTrue("F1-score not over 95%", eval.f1() > 0.95);

    } catch (IOException e){
        System.out.println("Could not load MNIST.");
    }
}
 

@Test
public void testJsonLossFunctions() {

    ILossFunction[] lossFunctions = new ILossFunction[] {new LossBinaryXENT(), new LossBinaryXENT(),
                    new LossCosineProximity(), new LossHinge(), new LossKLD(), new LossKLD(), new LossL1(),
                    new LossL1(), new LossL2(), new LossL2(), new LossMAE(), new LossMAE(), new LossMAPE(),
                    new LossMAPE(), new LossMCXENT(), new LossMSE(), new LossMSE(), new LossMSLE(), new LossMSLE(),
                    new LossNegativeLogLikelihood(), new LossNegativeLogLikelihood(), new LossPoisson(),
                    new LossSquaredHinge(), new LossFMeasure(), new LossFMeasure(2.0)};

    Activation[] outputActivationFn = new Activation[] {Activation.SIGMOID, //xent
                    Activation.SIGMOID, //xent
                    Activation.TANH, //cosine
                    Activation.TANH, //hinge -> trying to predict 1 or -1
                    Activation.SIGMOID, //kld -> probab so should be between 0 and 1
                    Activation.SOFTMAX, //kld + softmax
                    Activation.TANH, //l1
                    Activation.SOFTMAX, //l1 + softmax
                    Activation.TANH, //l2
                    Activation.SOFTMAX, //l2 + softmax
                    Activation.IDENTITY, //mae
                    Activation.SOFTMAX, //mae + softmax
                    Activation.IDENTITY, //mape
                    Activation.SOFTMAX, //mape + softmax
                    Activation.SOFTMAX, //mcxent
                    Activation.IDENTITY, //mse
                    Activation.SOFTMAX, //mse + softmax
                    Activation.SIGMOID, //msle  -   requires positive labels/activations due to log
                    Activation.SOFTMAX, //msle + softmax
                    Activation.SIGMOID, //nll
                    Activation.SOFTMAX, //nll + softmax
                    Activation.SIGMOID, //poisson - requires positive predictions due to log... not sure if this is the best option
                    Activation.TANH, //squared hinge
                    Activation.SIGMOID, //f-measure (binary, single sigmoid output)
                    Activation.SOFTMAX //f-measure (binary, 2-label softmax output)
    };

    int[] nOut = new int[] {1, //xent
                    3, //xent
                    5, //cosine
                    3, //hinge
                    3, //kld
                    3, //kld + softmax
                    3, //l1
                    3, //l1 + softmax
                    3, //l2
                    3, //l2 + softmax
                    3, //mae
                    3, //mae + softmax
                    3, //mape
                    3, //mape + softmax
                    3, //mcxent
                    3, //mse
                    3, //mse + softmax
                    3, //msle
                    3, //msle + softmax
                    3, //nll
                    3, //nll + softmax
                    3, //poisson
                    3, //squared hinge
                    1, //f-measure (binary, single sigmoid output)
                    2, //f-measure (binary, 2-label softmax output)
    };

    for (int i = 0; i < lossFunctions.length; i++) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.ADAM).list()
                        .layer(0, new DenseLayer.Builder().nIn(4).nOut(nOut[i]).activation(Activation.TANH).build())
                        .layer(1, new LossLayer.Builder().lossFunction(lossFunctions[i])
                                        .activation(outputActivationFn[i]).build())
                        .validateOutputLayerConfig(false).build();

        String json = conf.toJson();
        String yaml = conf.toYaml();

        MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json);
        MultiLayerConfiguration fromYaml = MultiLayerConfiguration.fromYaml(yaml);

        assertEquals(conf, fromJson);
        assertEquals(conf, fromYaml);
    }
}