org.nd4j.linalg.activations.IActivation Java Examples

public static IActivation activationInstance(String activation) {

        IActivation activationFn = new ActivationSigmoid();
        switch (activation) {
            case "identity":
                activationFn = new ActivationIdentity();
            case "softmax":
                activationFn = new ActivationSoftmax();
                break;
            case "sigmoid":
                activationFn = new ActivationSigmoid();
                break;
            case "tanh":
                activationFn = new ActivationTanH();
                break;
            case "reul":
                activationFn = new ActivationReLU();
                break;
        }

        return activationFn;

    }
 

protected INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    INDArray scoreArr = output.rsubi(labels);
    scoreArr = scoreArr.muli(scoreArr);

    //Weighted loss function
    if (weights != null) {
        if (weights.length() != output.size(1)) {
            throw new IllegalStateException("Weights vector (length " + weights.length()
                    + ") does not match output.size(1)=" + output.size(1));
        }
        scoreArr.muliRowVector(weights);
    }

    //Loss function with masking
    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    // Clip output and labels to be between Nd4j.EPS_THREsHOLD and 1, i.e. a valid non-zero probability
    output = Transforms.min(Transforms.max(output, Nd4j.EPS_THRESHOLD, false), 1, false);
    labels = Transforms.min(Transforms.max(labels, Nd4j.EPS_THRESHOLD, true), 1, false);

    INDArray logRatio = Transforms.log(output.rdivi(labels), false);

    INDArray scoreArr = logRatio.muli(labels);
    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

@Test
public void testForward() throws Exception {
    EnvironmentContext context = EnvironmentFactory.getContext();
    Future<?> task = context.doTask(() -> {
        INDArray array = Nd4j.linspace(-2.5D, 2.0D, 10).reshape(5, 2);
        IActivation oldFunction = getOldFunction();
        INDArray value = oldFunction.getActivation(array.dup(), true);

        DenseMatrix input = getMatrix(array);
        DenseMatrix output = DenseMatrix.valueOf(input.getRowSize(), input.getColumnSize());
        ActivationFunction newFuction = getNewFunction();
        newFuction.forward(input, output);

        System.out.println(value);
        System.out.println(output);
        Assert.assertTrue(equalMatrix(output, value));

        DenseVector vector = DenseVector.valueOf(input.getColumnSize());
        for (int index = 0, size = input.getRowSize(); index < size; index++) {
            newFuction.forward(input.getRowVector(index), vector);
            Assert.assertTrue(equalVector(vector, output.getRowVector(index)));
        }
    });
    task.get();
}
 

public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    /*
     mean of (yhat - y * log(yhat))
     */
    //INDArray postOutput = Nd4j.utioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup()));
    INDArray postOutput = activationFn.getActivation(preOutput.dup(), true);

    INDArray scoreArr = Transforms.log(postOutput);
    scoreArr.muli(labels);
    scoreArr = postOutput.sub(scoreArr);

    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    /* y_hat is -1 or 1
    hinge loss is max(0,1-y_hat*y)
     */
    //INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup()));
    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    INDArray scoreArr = output.muli(labels); //y*yhat
    scoreArr.rsubi(1.0); //1 - y*yhat

    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr; // 1 - y*yhat
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels, INDArray preOutput, IActivation activationFn,
                INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);
    double score = scoreArr.sumNumber().doubleValue();

    if (average) {
        score /= scoreArr.size(0);
    }

    return score;
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    INDArray actSubPredicted = labels.sub(output);
    INDArray dLda = Nd4j.getExecutioner().execAndReturn(new Sign(actSubPredicted));
    INDArray absLabels = Nd4j.getExecutioner().execAndReturn(new Abs(labels.dup()));
    dLda.divi(absLabels).muli(-100.0 / labels.size(1));

    //Weighted loss function
    if (weights != null) {
        dLda.muliRowVector(weights);
    }

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - but buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    INDArray gradient = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params

    if (mask != null) {
        LossUtil.applyMask(gradient, mask);
    }

    return gradient;
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    final INDArray grad = Nd4j.ones(labels.shape());
    calculate(labels, preOutput, activationFn, mask, null, grad);
    return grad;
}
 

@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                boolean average) {
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);

    double score = scoreArr.sumNumber().doubleValue();

    if (average)
        score /= scoreArr.size(0);

    return score;
}
 

@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                boolean average) {

    double[] d = computeScoreNumDenom(labels, preOutput, activationFn, mask, average);
    double numerator = d[0];
    double denominator = d[1];

    if (numerator == 0.0 && denominator == 0.0) {
        return 0.0;
    }

    return 1.0 - numerator / denominator;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels,
                INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

/**
 * Computes the aggregate score as a sum of all of the individual scores of
 * each of the labels against each of the outputs of the network.  For
 * the mixture density network, this is the negative log likelihood that
 * the given labels fall within the probability distribution described by
 * the mixture of gaussians of the network output.
 * @param labels Labels to score against the network.
 * @param preOutput Output of the network (before activation function has been called).
 * @param activationFn Activation function for the network.
 * @param mask Mask to be applied to labels (not used for MDN).
 * @param average Whether or not to return an average instead of a total score (not used).
 * @return Returns a single double which corresponds to the total score of all label values.
 */
@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                boolean average) {
    // The score overall consists of the
    // sum of the negative log likelihoods for each
    // of the individual labels.
    INDArray scoreArr = computeScoreArray(labels, preOutput, activationFn, mask);
    double score = scoreArr.sumNumber().doubleValue();
    if (average) {
        score /= scoreArr.size(0);
    }
    return score;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels, INDArray preOutput, IActivation activationFn,
                INDArray mask, boolean average) {
    double score = computeScore(labels, preOutput, activationFn, mask, average);
    INDArray gradient = computeGradient(labels, preOutput, activationFn, mask);
    Pair<Double, INDArray> returnCode = new Pair<>(score, gradient);
    return returnCode;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels,
                INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...
    //Yes - will implement in round two. Just want to get done now.

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    INDArray outSubLabels = output.sub(labels);
    INDArray dLda = Nd4j.getExecutioner().execAndReturn(new Sign(outSubLabels));

    if (weights != null) {
        dLda.muliRowVector(weights);
    }

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - but buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    //dL/dz
    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation function param gradients

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels, INDArray preOutput, IActivation activationFn,
                INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);

    INDArray bitMaskRowCol = scoreArr.dup();
    /*
        bit mask is 0 if 1-sigma(y*yhat) is neg, bit mask is 1 if 1-sigma(y*yhat) is +ve
     */
    BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0));
    BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0));

    INDArray dLda = scoreArr.muli(2).muli(labels.neg());
    dLda.muli(bitMaskRowCol);

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - though buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray yhat = activationFn.getActivation(preOutput.dup(), true);
    INDArray yL2norm = labels.norm2(1);

    INDArray yhatL2norm = yhat.norm2(1);
    INDArray yhatL2normSq = yhatL2norm.mul(yhatL2norm);

    //Note: This is not really the L1 norm since I am not taking abs values
    INDArray yhatDotyL1norm = labels.mul(yhat).sum(1);

    INDArray dLda = labels.mulColumnVector(yhatL2normSq);
    dLda.subi(yhat.mulColumnVector(yhatDotyL1norm));

    // transform vals to avoid nans before div
    yL2norm = Transforms.max(yL2norm, Nd4j.EPS_THRESHOLD, false);
    yhatL2norm = Transforms.max(yhatL2norm, Nd4j.EPS_THRESHOLD, false);
    yhatL2normSq = Transforms.max(yhatL2normSq, Nd4j.EPS_THRESHOLD, false);

    dLda.diviColumnVector(yL2norm);
    dLda.diviColumnVector(yhatL2norm.mul(yhatL2normSq));
    dLda.muli(-1);

    //dL/dz
    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO loss functions with params

    if (mask != null) {
        gradients.muliColumnVector(mask);
    }

    return gradients;
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }

    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    if(activationFn instanceof ActivationSoftmax && softmaxClipEps > 0.0){
        BooleanIndexing.replaceWhere(output, softmaxClipEps, Conditions.lessThan(softmaxClipEps));
        BooleanIndexing.replaceWhere(output, 1.0-softmaxClipEps, Conditions.greaterThan(1.0-softmaxClipEps));
    }
    INDArray scoreArr = Transforms.log(output, false).muli(labels);

    //Weighted loss function
    if (weights != null) {
        if (weights.length() != scoreArr.size(1)) {
            throw new IllegalStateException("Weights vector (length " + weights.length()
                            + ") does not match output.size(1)=" + preOutput.size(1));
        }
        scoreArr.muliRowVector(weights);
    }

    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    //INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup()));
    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    INDArray p1 = output.add(1.0);
    INDArray dlda = p1.rdiv(2.0 / labels.size(1));
    INDArray logRatio = Transforms.log(p1.divi(labels.add(1.0)), false);
    dlda.muli(logRatio);

    if (weights != null) {
        dlda.muliRowVector(weights);
    }

    if (mask != null && LossUtil.isPerOutputMasking(dlda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - though buy us a tiny bit of performance
        LossUtil.applyMask(dlda, mask);
    }

    //dL/dz
    INDArray gradients = activationFn.backprop(preOutput, dlda).getFirst(); //TODO activation functions with weights

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                           boolean average) {
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);

    double score = scoreArr.sumNumber().doubleValue();

    if (average)
        score /= scoreArr.size(0);

    return score;
}
 

@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                boolean average) {

    double score = super.computeScore(labels, preOutput, activationFn, mask, average);
    score /= (labels.size(1));
    return score;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels,
                INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...
    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels,
                INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...
    //Yes - will implement in round two. Just want to get done now.

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    INDArray yHat = activationFn.getActivation(preOutput.dup(), true);
    INDArray yDivyhat = labels.div(yHat);
    INDArray dLda = yDivyhat.rsubi(1);

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - though buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels,
                INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}
 

@Override
public double computeScore(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask,
                boolean average) {
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);

    double score = scoreArr.sumNumber().doubleValue();

    if (average)
        score /= scoreArr.size(0);

    return score;
}
 

@Override
public Pair<Double, INDArray> computeGradientAndScore(INDArray labels,
                INDArray preOutput, IActivation activationFn, INDArray mask, boolean average) {
    //TODO: probably a more efficient way to do this...

    return new Pair<>(computeScore(labels, preOutput, activationFn, mask, average),
                    computeGradient(labels, preOutput, activationFn, mask));
}