Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#muliColumnVector()

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    double[] d = computeScoreNumDenom(labels, preOutput, activationFn, mask, false);
    double numerator = d[0];
    double denominator = d[1];

    if (numerator == 0.0 && denominator == 0.0) {
        //Zero score -> zero gradient
        return Nd4j.create(preOutput.shape());
    }

    double secondTerm = numerator / (denominator * denominator);

    INDArray dLdOut;
    if (labels.size(1) == 1) {
        //Single binary output case
        dLdOut = labels.mul(1 + beta * beta).divi(denominator).subi(secondTerm);
    } else {
        //Softmax case: the getColumn(1) here is to account for the fact that we're using prob(class1)
        // only in the score function; column(1) is equivalent to output for the single output case
        dLdOut = Nd4j.create(labels.shape());
        dLdOut.getColumn(1).assign(labels.getColumn(1).mul(1 + beta * beta).divi(denominator).subi(secondTerm));
    }

    //Negate relative to description in paper, as we want to *minimize* 1.0-fMeasure, which is equivalent to
    // maximizing fMeasure
    dLdOut.negi();

    INDArray dLdPreOut = activationFn.backprop(preOutput, dLdOut).getFirst();

    if (mask != null) {
        dLdPreOut.muliColumnVector(mask);
    }

    return dLdPreOut;
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    INDArray yhat = activationFn.getActivation(preOutput.dup(), true);
    INDArray yL2norm = labels.norm2(1);

    INDArray yhatL2norm = yhat.norm2(1);
    INDArray yhatL2normSq = yhatL2norm.mul(yhatL2norm);

    //Note: This is not really the L1 norm since I am not taking abs values
    INDArray yhatDotyL1norm = labels.mul(yhat).sum(true,1);

    INDArray dLda = labels.mulColumnVector(yhatL2normSq);
    dLda.subi(yhat.mulColumnVector(yhatDotyL1norm));

    // transform vals to avoid nans before div
    yL2norm = Transforms.max(yL2norm, Nd4j.EPS_THRESHOLD, false);
    yhatL2norm = Transforms.max(yhatL2norm, Nd4j.EPS_THRESHOLD, false);
    yhatL2normSq = Transforms.max(yhatL2normSq, Nd4j.EPS_THRESHOLD, false);

    dLda.diviColumnVector(yL2norm);
    dLda.diviColumnVector(yhatL2norm.mul(yhatL2normSq));
    dLda.muli(-1);

    //dL/dz
    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO loss functions with params

    if (mask != null) {
        gradients.muliColumnVector(mask);
    }

    return gradients;
}
 

/**
 *
 * @param labels
 * @param preOutput
 * @param activationFn
 * @param mask
 * @return
 */
public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype

    /*
     mean of -(y.dot(yhat)/||y||*||yhat||)
     */
    INDArray postOutput = activationFn.getActivation(preOutput.dup(), true);

    INDArray yhatmag = postOutput.norm2(1);
    INDArray ymag = labels.norm2(1);
    yhatmag = Transforms.max(yhatmag, Nd4j.EPS_THRESHOLD, false);
    ymag = Transforms.max(ymag, Nd4j.EPS_THRESHOLD, false);

    INDArray scoreArr = postOutput.mul(labels);
    scoreArr.diviColumnVector(yhatmag);
    scoreArr.diviColumnVector(ymag);

    if (mask != null) {
        if (!mask.isColumnVector()) {
            //Per-output masking doesn't really make sense for cosine proximity
            throw new UnsupportedOperationException("Expected column vector mask array for LossCosineProximity."
                            + " Got mask array with shape " + Arrays.toString(mask.shape())
                            + "; per-output masking is not " + "supported for LossCosineProximity");
        }
        scoreArr.muliColumnVector(mask);
    }
    return scoreArr.muli(-1);
}
 

@Test
public void testInvalidColVectorOp1() {
    INDArray first = Nd4j.create(10, 10);
    INDArray col = Nd4j.create(5, 1);
    try {
        first.muliColumnVector(col);
        fail("Should have thrown IllegalStateException");
    } catch (IllegalStateException e) {
        //OK
    }
}
 

@Override
protected void applyMask(INDArray to) {
    //For output layers: can be either per-example masking, or per-
    if (maskArray.isColumnVectorOrScalar()) {
        to.muliColumnVector(maskArray.castTo(to.dataType()));
    } else if (Arrays.equals(to.shape(), maskArray.shape())) {
        to.muli(maskArray.castTo(to.dataType()));
    } else {
        throw new IllegalStateException("Invalid mask array: per-example masking should be a column vector, "
                + "per output masking arrays should be the same shape as the output/labels arrays. Mask shape: "
                + Arrays.toString(maskArray.shape()) + ", output shape: " + Arrays.toString(to.shape())
                + layerId());
    }
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    double[] d = computeScoreNumDenom(labels, preOutput, activationFn, mask, false);
    double numerator = d[0];
    double denominator = d[1];

    if (numerator == 0.0 && denominator == 0.0) {
        //Zero score -> zero gradient
        return Nd4j.create(preOutput.shape());
    }

    double secondTerm = numerator / (denominator * denominator);

    INDArray dLdOut;
    if (labels.size(1) == 1) {
        //Single binary output case
        dLdOut = labels.mul(1 + beta * beta).divi(denominator).subi(secondTerm);
    } else {
        //Softmax case: the getColumn(1) here is to account for the fact that we're using prob(class1)
        // only in the score function; column(1) is equivalent to output for the single output case
        dLdOut = Nd4j.create(labels.shape());
        dLdOut.getColumn(1).assign(labels.getColumn(1).mul(1 + beta * beta).divi(denominator).subi(secondTerm));
    }

    //Negate relative to description in paper, as we want to *minimize* 1.0-fMeasure, which is equivalent to
    // maximizing fMeasure
    dLdOut.negi();

    INDArray dLdPreOut = activationFn.backprop(preOutput, dLdOut).getFirst();

    if (mask != null) {
        dLdPreOut.muliColumnVector(mask);
    }

    return dLdPreOut;
}
 

@Test
public void testClipByNorm(){
    //Expected: if array.norm2(1) is less than 1.0, not modified
    //Otherwise: array.tad(x,1) = array.tad(x,1) * 1.0 / array.tad(x,1).norm2()

    Nd4j.getRandom().setSeed(12345);
    INDArray arr = Nd4j.rand(3,5);
    INDArray norm2_1 = arr.norm2(1);
    arr.diviColumnVector(norm2_1);

    norm2_1 = arr.norm2(1);
    assertEquals(Nd4j.ones(3), norm2_1);

    INDArray scale = Nd4j.create(new double[]{1.1, 1.0, 0.9}, new int[]{3});
    arr.muliColumnVector(scale);
    norm2_1 = arr.norm2(1);

    INDArray out = Nd4j.create(arr.shape());

    Nd4j.getExecutioner().exec(DynamicCustomOp.builder("clipbynorm")
            .addInputs(arr)
            .addOutputs(out)
            .addIntegerArguments(1)
            .addFloatingPointArguments(1.0)
            .build());

    INDArray norm2_1b = out.norm2(1);
    INDArray exp = Nd4j.create(new double[]{1.0, 1.0, norm2_1.getDouble(2)}, new int[]{3});

    assertEquals(exp, norm2_1b);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);

    INDArray ret;
    if(!training){
        ret = input;
    } else {
        if(layerConf().getIDropout() != null){
            INDArray result;
            if(inputModificationAllowed){
                result = input;
            } else {
                result = workspaceMgr.createUninitialized(ArrayType.INPUT, input.dataType(), input.shape(), input.ordering());
            }

            ret = layerConf().getIDropout().applyDropout(input, result, getIterationCount(), getEpochCount(), workspaceMgr);
        } else {
            ret = workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input);
        }
    }

    if (maskArray != null) {
        ret.muliColumnVector(maskArray);
    }

    ret = workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, ret);
    return ret;
}
 

/**
 *
 * @param to
 * @param mask
 */
public static void applyMask(INDArray to, INDArray mask) {
    //Two possibilities exist: it's *per example* masking, or it's *per output* masking
    //These cases have different mask shapes. Per example: column vector. Per output: same shape as score array
    if (mask.isColumnVectorOrScalar()) {
        to.muliColumnVector(mask);
    } else if (Arrays.equals(to.shape(), mask.shape())) {
        to.muli(mask);
    } else {
        throw new IllegalStateException("Invalid mask array: per-example masking should be a column vector, "
                        + "per output masking arrays should be the same shape as the labels array. Mask shape: "
                        + Arrays.toString(mask.shape()) + ", output shape: " + Arrays.toString(to.shape()));
    }
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    INDArray scoreArr;
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    INDArray yMinusyHat = Transforms.abs(labels.sub(output));
    scoreArr = yMinusyHat.mul(yMinusyHat);
    scoreArr = scoreArr.mul(maskData);

    if (mask != null) {
        scoreArr.muliColumnVector(mask);
    }
    return scoreArr;
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray input = this.input;
    if (input.rank() != 3)
        throw new UnsupportedOperationException(
                        "Input must be rank 3. Got input with rank " + input.rank() + " " + layerId());
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);

    applyDropOutIfNecessary(training, workspaceMgr);
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
    }
    INDArray input2d = TimeSeriesUtils.reshape3dTo2d(input.castTo(W.dataType()), workspaceMgr, ArrayType.FF_WORKING_MEM);

    INDArray act2d = layerConf().getActivationFn().getActivation(input2d.mmul(W).addiRowVector(b), training);
    if (maskArray != null) {
        if(!maskArray.isColumnVectorOrScalar() || Arrays.equals(maskArray.shape(), act2d.shape())){
            //Per output masking
            act2d.muli(maskArray.castTo(act2d.dataType()));
        } else {
            //Per time step masking
            act2d.muliColumnVector(maskArray.castTo(act2d.dataType()));
        }
    }

    INDArray ret = TimeSeriesUtils.reshape2dTo3d(act2d, input.size(0), workspaceMgr, ArrayType.ACTIVATIONS);
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        ret = ret.permute(0, 2, 1);
    }
    return ret;
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray z = input;
    INDArray ret = layerConf().getActivationFn().getActivation(z.dup(), training);

    if (maskArray != null) {
        ret.muliColumnVector(maskArray);
    }

    INDArray out = workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, ret);
    return out;
}
 

@Test
public void testInvalidColVectorOp1() {
    INDArray first = Nd4j.create(10, 10);
    INDArray col = Nd4j.create(5, 1);
    try {
        first.muliColumnVector(col);
        fail("Should have thrown IllegalStateException");
    } catch (IllegalStateException e) {
        //OK
    }
}
 

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoBackward())
        throw new IllegalStateException("Cannot do backward pass: error not set");

    INDArray a = inputs[0];
    INDArray b = inputs[1];
    INDArray out = doForward(tbptt, workspaceMgr);
    Transforms.max(out, eps, false); // in case of 0

    INDArray dLdlambda = epsilon; //dL/dlambda aka 'epsilon' - from layer above

    INDArray sNegHalf = out.rdiv(1.0); //s^(-1/2) = 1.0 / s^(1/2) = 1.0 / out

    INDArray diff;
    try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)){
        diff = a.sub(b);
    }

    INDArray first = dLdlambda.mul(sNegHalf); //Column vector for all cases

    INDArray dLda;
    INDArray dLdb;
    if (a.rank() == 2) {
        //2d case (MLPs etc)
        dLda = diff.muliColumnVector(first);
        try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)) {
            dLdb = dLda.neg();
        }
    } else {
        //RNN and CNN case - Broadcast along dimension 0
        dLda = Nd4j.getExecutioner().exec(new BroadcastMulOp(diff, first, diff, 0));
        try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)) {
            dLdb = dLda.neg();
        }
    }

    return new Pair<>(null, new INDArray[] {dLda, dLdb});
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    INDArray yMinusyHat = labels.sub(output);
    INDArray dldyhat = yMinusyHat.mul(-2);

    INDArray gradients = activationFn.backprop(preOutput.dup(), dldyhat).getFirst();
    gradients = gradients.mul(maskData);
    // multiply with masks, always
    if (mask != null) {
        gradients.muliColumnVector(mask);
    }

    return gradients;
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    INDArray scoreArr;
    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    INDArray yMinusyHat = Transforms.abs(labels.sub(output));
    scoreArr = yMinusyHat.mul(yMinusyHat);
    scoreArr = scoreArr.mul(maskData);

    if (mask != null) {
        scoreArr.muliColumnVector(mask);
    }
    return scoreArr;
}
 

/**
 *
 * @param to
 * @param mask
 */
public static void applyMask(INDArray to, INDArray mask) {
    //Two possibilities exist: it's *per example* masking, or it's *per output* masking
    //These cases have different mask shapes. Per example: column vector. Per output: same shape as score array
    if (mask.isColumnVectorOrScalar()) {
        to.muliColumnVector(mask.castTo(to.dataType()));
    } else if (Arrays.equals(to.shape(), mask.shape())) {
        to.muli(mask.castTo(to.dataType()));
    } else {
        throw new IllegalStateException("Invalid mask array: per-example masking should be a column vector, "
                        + "per output masking arrays should be the same shape as the labels array. Mask shape: "
                        + Arrays.toString(mask.shape()) + ", output shape: " + Arrays.toString(to.shape()));
    }
}
 

protected void applyMask(INDArray to) {
    to.muliColumnVector(maskArray.castTo(to.dataType()));
}
 

@Test
public void testClipByNorm2(){
    //Expected: if array.norm2(1) is less than 1.0, not modified
    //Otherwise: array.tad(x,1) = array.tad(x,1) * 1.0 / array.tad(x,1).norm2()

    Nd4j.getRandom().setSeed(12345);
    INDArray arr = Nd4j.rand(3,5);
    INDArray norm2_1 = arr.norm2(1);
    arr.diviColumnVector(norm2_1);

    norm2_1 = arr.norm2(1);
    assertEquals(Nd4j.ones(3), norm2_1);

    INDArray scale = Nd4j.create(new double[]{1.1, 1.0, 0.9}, new int[]{3,1});
    arr.muliColumnVector(scale);
    norm2_1 = arr.norm2(1);

    INDArray out = Nd4j.createUninitialized(arr.shape());

    OpTestCase op = new OpTestCase(DynamicCustomOp.builder("clipbynorm")
            .addInputs(arr)
            .addOutputs(out)
            .addIntegerArguments(1)
            .addFloatingPointArguments(1.0)
            .build());

    INDArray expNorm2 = Nd4j.create(new double[]{1.0, 1.0, norm2_1.getDouble(2)}, new int[]{3,1});

    INDArray expOut = arr.divColumnVector(norm2_1).muliColumnVector(expNorm2);
    op.expectedOutput(0, expOut);

    System.out.println("Input");
    System.out.println(arr.shapeInfoToString());
    System.out.println(Arrays.toString(arr.data().asFloat()));

    System.out.println("Expected");
    System.out.println(expOut.shapeInfoToString());
    System.out.println(Arrays.toString(expOut.data().asFloat()));

    String err = OpValidation.validate(op);
    assertNull(err);
}
 

public Construct3dDataSet(INDArray featureScale, int timeSteps, int samples, int origin) {
    this.featureScale = featureScale;
    this.timeSteps = timeSteps;
    this.samples = samples;
    this.origin = origin;

    // FIXME: int cast
    numFeatures = (int) featureScale.size(0);
    maxN = samples * timeSteps;
    INDArray template = Nd4j.linspace(origin, origin + timeSteps - 1, timeSteps);
    template = Nd4j.concat(0, Nd4j.linspace(origin, origin + timeSteps - 1, timeSteps), template);
    template = Nd4j.concat(0, Nd4j.linspace(origin, origin + timeSteps - 1, timeSteps), template);
    template.muliColumnVector(featureScale);
    template = template.reshape(1, numFeatures, timeSteps);
    INDArray featureMatrix = template.dup();

    int newStart = origin + timeSteps;
    int newEnd;
    for (int i = 1; i < samples; i++) {
        newEnd = newStart + timeSteps - 1;
        template = Nd4j.linspace(newStart, newEnd, timeSteps);
        template = Nd4j.concat(0, Nd4j.linspace(newStart, newEnd, timeSteps), template);
        template = Nd4j.concat(0, Nd4j.linspace(newStart, newEnd, timeSteps), template);
        template.muliColumnVector(featureScale);
        template = template.reshape(1, numFeatures, timeSteps);
        newStart = newEnd + 1;
        featureMatrix = Nd4j.concat(0, featureMatrix, template);
    }
    INDArray labelSet = featureMatrix.dup();
    this.newOrigin = newStart;
    sampleDataSet = new DataSet(featureMatrix, labelSet);

    //calculating stats
    // The theoretical mean should be the mean of 1,..samples*timesteps
    float theoreticalMean = origin - 1 + (samples * timeSteps + 1) / 2.0f;
    expectedMean = Nd4j.create(new double[] {theoreticalMean, theoreticalMean, theoreticalMean}).reshape(3, 1);
    expectedMean.muliColumnVector(featureScale);

    float stdNaturalNums = (float) Math.sqrt((samples * samples * timeSteps * timeSteps - 1) / 12);
    expectedStd = Nd4j.create(new float[] {stdNaturalNums, stdNaturalNums, stdNaturalNums}).reshape(3, 1);
    expectedStd.muliColumnVector(Transforms.abs(featureScale, true));
    //preprocessors use the population std so divides by n not (n-1)
    expectedStd = expectedStd.dup().muli(Math.sqrt(maxN)).divi(Math.sqrt(maxN)).transpose();

    //min max assumes all scaling values are +ve
    expectedMin = Nd4j.ones(3, 1).muliColumnVector(featureScale);
    expectedMax = Nd4j.ones(3, 1).muli(samples * timeSteps).muliColumnVector(featureScale);
}