org.nd4j.linalg.lossfunctions.ILossFunction Java Examples

/** Compute score after labels and input have been set.
 * @param fullNetRegTerm Regularization score term for the entire network
 * @param training whether score should be calculated at train or test time (this affects things like application of
 *                 dropout, etc)
 * @return score (loss function)
 */
@Override
public double computeScore(double fullNetRegTerm, boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (input == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = preOutput2d(training, workspaceMgr);

    ILossFunction lossFunction = layerConf().getLossFn();

    double score = lossFunction.computeScore(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM), preOut,
            layerConf().getActivationFn(), maskArray,false);
    if(conf().isMiniBatch())
        score /= getInputMiniBatchSize();

    score += fullNetRegTerm;
    this.score = score;
    return score;
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    //For RNN: need to sum up the score over each time step before returning.

    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = preOutput2d(false, workspaceMgr);

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
            lossFunction.computeScoreArray(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM), preOut,
                    layerConf().getActivationFn(), maskArray);
    INDArray summedScores = scoreArray.sum(1);

    if (fullNetRegTerm != 0.0) {
        summedScores.addi(fullNetRegTerm);
    }

    return summedScores;
}
 

private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    // INDArray delta = lossFunction.computeGradient(labels2d, preOut,
    // layerConf().getActivationFunction(), maskArray);
    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.gemm(input, delta, weightGradView, true, false, 1.0, 0.0); // Equivalent to: weightGradView.assign(input.transpose().mmul(delta));
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradView);

    if (hasBias()) {
        INDArray biasGradView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGradView, 0); // biasGradView is initialized/zeroed first in sum op
        gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradView);
    }

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

/** Compute score after labels and input have been set.
 * @param fullNetRegTerm Regularization score term for the entire network
 * @param training whether score should be calculated at train or test time (this affects things like application of
 *                 dropout, etc)
 * @return score (loss function)
 */
@Override
public double computeScore(double fullNetRegTerm, boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    this.fullNetworkRegularizationScore = fullNetRegTerm;
    INDArray preOut = input;

    ILossFunction lossFunction = layerConf().getLossFn();

    //double score = lossFunction.computeScore(getLabels2d(), preOut, layerConf().getActivationFunction(), maskArray, false);
    double score = lossFunction.computeScore(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray,
                    false);
    score /= getInputMiniBatchSize();
    score += fullNetworkRegularizationScore;

    this.score = score;
    return score;
}
 

protected void handleLossBackwardCompatibility(BaseOutputLayer baseLayer, ObjectNode on){
    if(baseLayer.getLossFn() == null && on.has("activationFunction")) {
        String lfn = on.get("lossFunction").asText();
        ILossFunction loss = null;
        switch (lfn) {
            case "MCXENT":
                loss = new LossMCXENT();
                break;
            case "MSE":
                loss = new LossMSE();
                break;
            case "NEGATIVELOGLIKELIHOOD":
                loss = new LossNegativeLogLikelihood();
                break;
            case "SQUARED_LOSS":
                loss = new LossL2();
                break;
            case "XENT":
                loss = new LossBinaryXENT();
        }
        baseLayer.setLossFn(loss);
    }
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    //For RNN: need to sum up the score over each time step before returning.

    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = preOutput2d(false, workspaceMgr);

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
                    lossFunction.computeScoreArray(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM), preOut,
                            layerConf().getActivationFn(), maskArray);
    //scoreArray: shape [minibatch*timeSeriesLength, 1]
    //Reshape it to [minibatch, timeSeriesLength] then sum over time step

    INDArray scoreArrayTs = TimeSeriesUtils.reshapeVectorToTimeSeriesMask(scoreArray, (int)input.size(0));
    INDArray summedScores = scoreArrayTs.sum(true, 1);

    if (fullNetRegTerm != 0.0) {
        summedScores.addi(fullNetRegTerm);
    }

    return summedScores;
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = preOutput2d(false, workspaceMgr);

    // calculate the intra-class score component
    INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
    INDArray centersForExamples = labels.mmul(centers);
    INDArray intraClassScoreArray = input.sub(centersForExamples);

    // calculate the inter-class score component
    ILossFunction interClassLoss = layerConf().getLossFn();
    INDArray scoreArray = interClassLoss.computeScoreArray(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM), preOut, layerConf().getActivationFn(),
                    maskArray);
    scoreArray.addi(intraClassScoreArray.muli(layerConf().getLambda() / 2));

    if (fullNetRegTerm != 0.0) {
        scoreArray.addi(fullNetRegTerm);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, scoreArray);
}
 

/** Compute score after labels and input have been set.
 * @param fullNetRegTerm Regularization score term for the entire network
 * @param training whether score should be calculated at train or test time (this affects things like application of
 *                 dropout, etc)
 * @return score (loss function)
 */
@Override
public double computeScore(double fullNetRegTerm, boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    this.fullNetRegTerm = fullNetRegTerm;
    INDArray preOut = preOutput2d(training, workspaceMgr);

    ILossFunction lossFunction = layerConf().getLossFn();

    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM);
    double score = lossFunction.computeScore(labels2d, preOut,
            layerConf().getActivationFn(), maskArray,false);

    if(conf().isMiniBatch())
        score /= getInputMiniBatchSize();

    score += fullNetRegTerm;

    this.score = score;
    return score;
}
 

@Override
public double computeScore(double fullNetRegTerm, boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray input = this.input;
    INDArray labels = this.labels;
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
        labels = input.permute(0, 2, 1);
    }
    INDArray input2d = TimeSeriesUtils.reshape3dTo2d(input, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray labels2d = TimeSeriesUtils.reshape3dTo2d(labels, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray maskReshaped;
    if(this.maskArray != null){
        if(this.maskArray.rank() == 3){
            maskReshaped = TimeSeriesUtils.reshapePerOutputTimeSeriesMaskTo2d(this.maskArray, workspaceMgr, ArrayType.FF_WORKING_MEM);
        } else {
            maskReshaped = TimeSeriesUtils.reshapeTimeSeriesMaskToVector(this.maskArray, workspaceMgr, ArrayType.FF_WORKING_MEM);
        }
    } else {
        maskReshaped = null;
    }

    ILossFunction lossFunction = layerConf().getLossFn();

    double score = lossFunction.computeScore(labels2d, input2d.dup(), layerConf().getActivationFn(), maskReshaped,false);
    score /= getInputMiniBatchSize();
    score += fullNetRegTerm;

    this.score = score;

    return score;
}
 

/** Returns tuple: {Gradient,Delta,Output} given preOut */
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    // delta calculation
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray delta = lossFunction.computeGradient(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray);

    // grab the empty gradient
    Gradient gradient = new DefaultGradient();

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

@Test
    public void testScore() throws Exception {
        EnvironmentContext context = EnvironmentFactory.getContext();
        Future<?> task = context.doTask(() -> {
            LinkedList<KeyValue<IActivation, ActivationFunction>> activetionList = new LinkedList<>();
            activetionList.add(new KeyValue<>(new ActivationSigmoid(), new SigmoidActivationFunction()));
//            activetionList.add(new KeyValue<>(new ActivationSoftmax(), new SoftMaxActivationFunction()));
            for (KeyValue<IActivation, ActivationFunction> keyValue : activetionList) {
                INDArray array = Nd4j.linspace(-2.5D, 2.0D, 10).reshape(5, 2);
                INDArray marks = Nd4j.create(new double[] { 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D }).reshape(5, 2);
                ILossFunction oldFunction = getOldFunction(marks);
                double value = oldFunction.computeScore(marks, array.dup(), keyValue.getKey(), null, false);

                Nd4jMatrix input = getMatrix(array.dup());
                Nd4jMatrix output = new Nd4jMatrix(Nd4j.zeros(input.getRowSize(), input.getColumnSize()));
                ActivationFunction function = keyValue.getValue();
                function.forward(input, output);
                LossFunction newFunction = getNewFunction(marks, function);
                newFunction.doCache(getMatrix(marks), output);
                double score = newFunction.computeScore(getMatrix(marks), output, null);

                System.out.println(value);
                System.out.println(score);

                if (Math.abs(value - score) > MathUtility.EPSILON) {
                    Assert.fail();
                }
            }
        });
        task.get();
    }
 

@Test
    public void testGradient() throws Exception {
        EnvironmentContext context = EnvironmentFactory.getContext();
        Future<?> task = context.doTask(() -> {
            LinkedList<KeyValue<IActivation, ActivationFunction>> activetionList = new LinkedList<>();
            activetionList.add(new KeyValue<>(new ActivationSigmoid(), new SigmoidActivationFunction()));
//            activetionList.add(new KeyValue<>(new ActivationSoftmax(), new SoftMaxActivationFunction()));
            for (KeyValue<IActivation, ActivationFunction> keyValue : activetionList) {
                INDArray array = Nd4j.linspace(-2.5D, 2.0D, 10).reshape(5, 2);
                INDArray marks = Nd4j.create(new double[] { 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D }).reshape(5, 2);
                ILossFunction oldFunction = getOldFunction(marks);
                INDArray value = oldFunction.computeGradient(marks, array.dup(), keyValue.getKey(), null);

                Nd4jMatrix input = getMatrix(array.dup());
                Nd4jMatrix output = new Nd4jMatrix(Nd4j.zeros(input.getRowSize(), input.getColumnSize()));
                ActivationFunction function = keyValue.getValue();
                function.forward(input, output);
                Nd4jMatrix gradient = new Nd4jMatrix(Nd4j.zeros(input.getRowSize(), input.getColumnSize()));
                LossFunction newFunction = getNewFunction(marks, function);
                newFunction.doCache(getMatrix(marks), output);
                newFunction.computeGradient(getMatrix(marks), output, null, gradient);
                function.backward(input, gradient, output);
                System.out.println(value);
                System.out.println(output);
                Assert.assertTrue(equalMatrix(output, value));
            }
        });
        task.get();
    }
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

/**
 * Get a mapper (minus general config) suitable for loading old format JSON - 1.0.0-alpha and before
 * @return Object mapper
 */
public static ObjectMapper getMapper100alpha(){
    //After 1.0.0-alpha, we switched from wrapper object to @class for subtype information
    ObjectMapper om = new ObjectMapper();

    om.addMixIn(InputPreProcessor.class, InputPreProcessorMixin.class);
    om.addMixIn(GraphVertex.class, GraphVertexMixin.class);
    om.addMixIn(Layer.class, LayerMixin.class);
    om.addMixIn(ReconstructionDistribution.class, ReconstructionDistributionMixin.class);
    om.addMixIn(IActivation.class, IActivationMixin.class);
    om.addMixIn(ILossFunction.class, ILossFunctionMixin.class);

    return om;
}
 

private INDArray getScoreArray(ReconstructionDistribution reconstructionDistribution, INDArray dataSubset,
                INDArray reconstructionSubset) {
    if (reconstructionDistribution instanceof LossFunctionWrapper) {
        ILossFunction lossFunction = ((LossFunctionWrapper) reconstructionDistribution).getLossFunction();
        //Re: the activation identity here - the reconstruction array already has the activation function applied,
        // so we don't want to apply it again. i.e., we are passing the output, not the pre-output.
        return lossFunction.computeScoreArray(dataSubset, reconstructionSubset, new ActivationIdentity(), null);
    } else if (reconstructionDistribution instanceof CompositeReconstructionDistribution) {
        return ((CompositeReconstructionDistribution) reconstructionDistribution)
                        .computeLossFunctionScoreArray(dataSubset, reconstructionSubset);
    } else {
        throw new UnsupportedOperationException("Cannot calculate composite reconstruction distribution");
    }
}
 

/**
 * Validate the output layer (or loss layer) configuration, to detect invalid consfiugrations. A DL4JInvalidConfigException
 * will be thrown for invalid configurations (like softmax + nOut=1).<br>
 * <p>
 * If the specified layer is not an output layer, this is a no-op
 *
 * @param layerName    Name of the layer
 * @param nOut         Number of outputs for the layer
 * @param isLossLayer  Should be true for loss layers (no params), false for output layers
 * @param activation   Activation function
 * @param lossFunction Loss function
 */
public static void validateOutputLayerConfiguration(String layerName, long nOut, boolean isLossLayer, IActivation activation, ILossFunction lossFunction){
    //nOut = 1 + softmax
    if(!isLossLayer && nOut == 1 && activation instanceof ActivationSoftmax){   //May not have valid nOut for LossLayer
        throw new DL4JInvalidConfigException("Invalid output layer configuration for layer \"" + layerName + "\": Softmax + nOut=1 networks " +
                "are not supported. Softmax cannot be used with nOut=1 as the output will always be exactly 1.0 " +
                "regardless of the input. " + COMMON_MSG);
    }

    //loss function required probability, but activation is outside 0-1 range
    if(lossFunctionExpectsProbability(lossFunction) && activationExceedsZeroOneRange(activation, isLossLayer)){
        throw new DL4JInvalidConfigException("Invalid output layer configuration for layer \"" + layerName + "\": loss function " + lossFunction +
                " expects activations to be in the range 0 to 1 (probabilities) but activation function " + activation +
                " does not bound values to this 0 to 1 range. This indicates a likely invalid network configuration. " + COMMON_MSG);
    }

    //Common mistake: softmax + xent
    if(activation instanceof ActivationSoftmax && lossFunction instanceof LossBinaryXENT){
        throw new DL4JInvalidConfigException("Invalid output layer configuration for layer \"" + layerName + "\": softmax activation function in combination " +
                "with LossBinaryXENT (binary cross entropy loss function). For multi-class classification, use softmax + " +
                "MCXENT (multi-class cross entropy); for binary multi-label classification, use sigmoid + XENT. " + COMMON_MSG);
    }

    //Common mistake: sigmoid + mcxent
    if(activation instanceof ActivationSigmoid && lossFunction instanceof LossMCXENT){
        throw new DL4JInvalidConfigException("Invalid output layer configuration for layer \"" + layerName + "\": sigmoid activation function in combination " +
                "with LossMCXENT (multi-class cross entropy loss function). For multi-class classification, use softmax + " +
                "MCXENT (multi-class cross entropy); for binary multi-label classification, use sigmoid + XENT. " + COMMON_MSG);
    }
}
 

@Test
@Override
public void testScore() throws Exception {
    EnvironmentContext context = EnvironmentFactory.getContext();
    Future<?> task = context.doTask(() -> {
        LinkedList<KeyValue<IActivation, ActivationFunction>> activetionList = new LinkedList<>();
        activetionList.add(new KeyValue<>(new ActivationSigmoid(), new SigmoidActivationFunction()));
        activetionList.add(new KeyValue<>(new ActivationSoftmax(), new SoftMaxActivationFunction()));
        for (KeyValue<IActivation, ActivationFunction> keyValue : activetionList) {
            INDArray array = Nd4j.linspace(-2.5D, 2.0D, 20).reshape(5, 4);
            INDArray marks = Nd4j.create(new double[] { 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D }).reshape(5, 2);
            ILossFunction oldFunction = getOldFunction();
            float value = (float) oldFunction.computeScore(marks, array.dup(), keyValue.getKey(), null, false);

            MathMatrix input = getMatrix(array.rows(), array.columns()).copyMatrix(getMatrix(array), false);
            MathMatrix output = getMatrix(input.getRowSize(), input.getColumnSize());
            ActivationFunction function = keyValue.getValue();
            function.forward(input, output);
            LossFunction newFunction = getNewFunction(function);
            newFunction.doCache(getMatrix(marks.rows(), marks.columns()).copyMatrix(getMatrix(marks), false), output);
            float score = newFunction.computeScore(getMatrix(marks.rows(), marks.columns()).copyMatrix(getMatrix(marks), false), output, null);

            System.out.println(value);
            System.out.println(score);
            if (!MathUtility.equal(value, score)) {
                Assert.fail();
            }
        }
    });
    task.get();
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    if (input.rank() != 4)
        throw new UnsupportedOperationException(
                "Input is not rank 4. Got input with rank " + input.rank() + " " + layerId() + " with shape "
                        + Arrays.toString(input.shape()) + " - expected shape " + layerConf().getFormat().dimensionNames());
    if (labels == null)
        throw new IllegalStateException("Labels are not set (null)");

    Preconditions.checkState(input.equalShapes(labels), "Input and label arrays do not have same shape: %ndShape vs. %ndShape",input, labels);

    CNN2DFormat format = layerConf().getFormat();
    INDArray input2d = ConvolutionUtils.reshape4dTo2d(input, format, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray labels2d = ConvolutionUtils.reshape4dTo2d(labels, format, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray maskReshaped = ConvolutionUtils.reshapeMaskIfRequired(maskArray, input, format, workspaceMgr, ArrayType.FF_WORKING_MEM);

    // delta calculation
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray delta2d = lossFunction.computeGradient(labels2d, input2d.dup(input2d.ordering()), layerConf().getActivationFn(), maskReshaped);
    delta2d = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta2d);

    INDArray delta4d = ConvolutionUtils.reshape2dTo4d(delta2d, input.shape(), format, workspaceMgr, ArrayType.ACTIVATION_GRAD);

    // grab the empty gradient
    Gradient gradient = new DefaultGradient();
    return new Pair<>(gradient, delta4d);
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = input;

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
                    lossFunction.computeScoreArray(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray);
    if (fullNetRegTerm != 0.0) {
        scoreArray.addi(fullNetRegTerm);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, scoreArray);
}
 

@Test
public void testScore() throws Exception {
    EnvironmentContext context = EnvironmentFactory.getContext();
    Future<?> task = context.doTask(() -> {
        LinkedList<KeyValue<IActivation, ActivationFunction>> activetionList = new LinkedList<>();
        activetionList.add(new KeyValue<>(new ActivationSigmoid(), new SigmoidActivationFunction()));
        activetionList.add(new KeyValue<>(new ActivationSoftmax(), new SoftMaxActivationFunction()));
        for (KeyValue<IActivation, ActivationFunction> keyValue : activetionList) {
            INDArray array = Nd4j.linspace(-2.5D, 2.0D, 10).reshape(5, 2);
            INDArray marks = Nd4j.create(new double[] { 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D }).reshape(5, 2);
            ILossFunction oldFunction = getOldFunction();
            double value = oldFunction.computeScore(marks, array.dup(), keyValue.getKey(), null, false);

            DenseMatrix input = getMatrix(array);
            DenseMatrix output = DenseMatrix.valueOf(input.getRowSize(), input.getColumnSize());
            ActivationFunction function = keyValue.getValue();
            function.forward(input, output);
            LossFunction newFunction = getNewFunction(function);
            newFunction.doCache(getMatrix(marks), output);
            double score = newFunction.computeScore(getMatrix(marks), output, null);

            System.out.println(value);
            System.out.println(score);

            if (Math.abs(value - score) > MathUtility.EPSILON) {
                Assert.fail();
            }
        }
    });
    task.get();
}
 

@Test
public void testGradient() throws Exception {
    EnvironmentContext context = EnvironmentFactory.getContext();
    Future<?> task = context.doTask(() -> {
        LinkedList<KeyValue<IActivation, ActivationFunction>> activetionList = new LinkedList<>();
        activetionList.add(new KeyValue<>(new ActivationSigmoid(), new SigmoidActivationFunction()));
        activetionList.add(new KeyValue<>(new ActivationSoftmax(), new SoftMaxActivationFunction()));
        for (KeyValue<IActivation, ActivationFunction> keyValue : activetionList) {
            INDArray array = Nd4j.linspace(-2.5D, 2.0D, 10).reshape(5, 2);
            INDArray marks = Nd4j.create(new double[] { 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D, 0D, 1D }).reshape(5, 2);
            ILossFunction oldFunction = getOldFunction();
            INDArray value = oldFunction.computeGradient(marks, array.dup(), keyValue.getKey(), null);

            DenseMatrix input = getMatrix(array);
            DenseMatrix output = DenseMatrix.valueOf(input.getRowSize(), input.getColumnSize());
            ActivationFunction function = keyValue.getValue();
            function.forward(input, output);
            DenseMatrix gradient = DenseMatrix.valueOf(input.getRowSize(), input.getColumnSize());
            LossFunction newFunction = getNewFunction(function);
            newFunction.doCache(getMatrix(marks), output);
            newFunction.computeGradient(getMatrix(marks), output, null, gradient);
            function.backward(input, gradient, output);
            System.out.println(value);
            System.out.println(output);
            Assert.assertTrue(equalMatrix(output, value));
        }
    });
    task.get();
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

/** Compute score after labels and input have been set.
 * @param fullNetRegTerm Regularization score term for the entire network
 * @param training whether score should be calculated at train or test time (this affects things like application of
 *                 dropout, etc)
 * @return score (loss function)
 */
@Override
public double computeScore(double fullNetRegTerm, boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    this.fullNetRegTerm = fullNetRegTerm;
    INDArray preOut = preOutput2d(training, workspaceMgr);

    // center loss has two components
    // the first enforces inter-class dissimilarity, the second intra-class dissimilarity (squared l2 norm of differences)
    ILossFunction interClassLoss = layerConf().getLossFn();

    // calculate the intra-class score component
    INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
    INDArray l = labels.castTo(centers.dataType()); //Ensure correct dtype (same as params); no-op if already correct dtype
    INDArray centersForExamples = l.mmul(centers);

    //        double intraClassScore = intraClassLoss.computeScore(centersForExamples, input, Activation.IDENTITY.getActivationFunction(), maskArray, false);
    INDArray norm2DifferenceSquared = input.sub(centersForExamples).norm2(1);
    norm2DifferenceSquared.muli(norm2DifferenceSquared);

    double sum = norm2DifferenceSquared.sumNumber().doubleValue();
    double lambda = layerConf().getLambda();
    double intraClassScore = lambda / 2.0 * sum;

    //        intraClassScore = intraClassScore * layerConf().getLambda() / 2;

    // now calculate the inter-class score component
    double interClassScore = interClassLoss.computeScore(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM), preOut, layerConf().getActivationFn(),
                    maskArray, false);

    double score = interClassScore + intraClassScore;

    score /= getInputMiniBatchSize();
    score += fullNetRegTerm;

    this.score = score;
    return score;
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

@Override
protected void setScoreWithZ(INDArray z) {
    if (input == null || z == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    ILossFunction lossFunction = layerConf().getLossFunction().getILossFunction();

    //double score = lossFunction.computeScore(input, z, layerConf().getActivationFunction(), maskArray, false);
    double score = lossFunction.computeScore(input, z, layerConf().getActivationFn(), maskArray, false);
    score /= getInputMiniBatchSize();
    score += calcRegularizationScore(false);

    this.score = score;
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}
 

@OptionMetadata(
    displayName = "loss function",
    description = "The loss function to use (default = LossMCXENT).",
    commandLineParamName = "lossFn",
    commandLineParamSynopsis = "-lossFn <specification>",
    displayOrder = 1
)
public LossFunction<? extends ILossFunction> getLossFn() {
  return LossFunction.create(backend.getLossFn());
}