Java Code Examples for org.nd4j.linalg.factory.Broadcast#mul()

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr){
    if (getRnnDataFormat() == RNNFormat.NWC){
        this.input = input.permute(0, 2, 1);
    }
    INDArray act4d = super.activate(training, workspaceMgr);
    INDArray act3d = act4d.reshape(act4d.size(0), act4d.size(1), act4d.size(2));

    if(maskArray != null){
        INDArray maskOut = feedForwardMaskArray(maskArray, MaskState.Active, (int)act3d.size(0)).getFirst();
        Preconditions.checkState(act3d.size(0) == maskOut.size(0) && act3d.size(2) == maskOut.size(1),
                "Activations dimensions (0,2) and mask dimensions (0,1) don't match: Activations %s, Mask %s",
                act3d.shape(), maskOut.shape());
        Broadcast.mul(act3d, maskOut, act3d, 0, 2);
    }
    if (getRnnDataFormat() == RNNFormat.NWC){
        this.input = input.permute(0, 2, 1);
        act3d = act3d.permute(0, 2, 1);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, act3d);   //Should be zero copy most of the time
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray rows = preOutput(training, workspaceMgr);

    INDArray ret = layerConf().getActivationFn().getActivation(rows, training);
    if (maskArray != null) {
        if(maskArray.rank() != 2 ||
                (input.rank() == 2 && !maskArray.equalShapes(input)) ||
                (input.rank() == 3 && (input.size(0) != maskArray.size(0) || input.size(2) != maskArray.size(1)))){
            throw new IllegalStateException("Mask array for EmbeddingSequenceLayer (when defined) must be rank 2 and" +
                    "have shape equal to input shape (when input is rank 2, shape [mb,tsLength]) or equal to input dimensions 0 and" +
                    " 2 (when input is rank 3, shape [mb,1,tsLength]). Input shape: " + Arrays.toString(input.shape()) +
                    ", mask shape: " + Arrays.toString(maskArray.shape()));
        }
        boolean ncw = layerConf().getOutputFormat() == RNNFormat.NCW;
        if(ncw){
            //Returned array: rank 3, shape [mb, vector, seqLength]. mask shape: [mb, seqLength]
            Broadcast.mul(ret, maskArray.castTo(ret.dataType()), ret, 0, 2);
        } else {
            //Returned array: rank 3, shape [mb, seqLength, vector]. mask shape: [mb, seqLength]
            Broadcast.mul(ret, maskArray.castTo(ret.dataType()), ret, 0, 1);
        }
    }
    return ret;
}
 

@Override
public void apply(INDArray param) {
    INDArray norm = param.norm2(dimensions);
    INDArray clipped = norm.unsafeDuplication();
    CustomOp op = DynamicCustomOp.builder("clipbyvalue")
            .addInputs(clipped)
            .callInplace(true)
            .addFloatingPointArguments(min, max)
            .build();
    Nd4j.getExecutioner().exec(op);

    norm.addi(epsilon);
    clipped.divi(norm);

    if(rate != 1.0){
        clipped.muli(rate).addi(norm.muli(1.0-rate));
    }

    Broadcast.mul(param, clipped, param, getBroadcastDims(dimensions, param.rank()) );
}
 

@Override
public INDArray applyDropout(INDArray inputActivations, INDArray output, int iteration, int epoch, LayerWorkspaceMgr workspaceMgr) {
    Preconditions.checkArgument(inputActivations.rank() == 5 || inputActivations.rank() == 4
            || inputActivations.rank() == 3, "Cannot apply spatial dropout to activations of rank %s: " +
            "spatial dropout can only be used for rank 3, 4 or 5 activations (input activations shape: %s)"
            , inputActivations.rank(), inputActivations.shape());

    double currP;
    if (pSchedule != null) {
        currP = pSchedule.valueAt(iteration, epoch);
    } else {
        currP = p;
    }

    val minibatch = inputActivations.size(0);
    val dim1 = inputActivations.size(1);
    mask = workspaceMgr.createUninitialized(ArrayType.INPUT, output.dataType(), minibatch, dim1).assign(1.0);
    Nd4j.getExecutioner().exec(new DropOutInverted(mask, currP));

    Broadcast.mul(inputActivations, mask, output, 0, 1);
    return output;
}
 

@Test
    public void testYoloStyle(){
        WorkspaceConfiguration WS_ALL_LAYERS_ACT_CONFIG = WorkspaceConfiguration.builder()
                .initialSize(0)
                .overallocationLimit(0.05)
                .policyLearning(LearningPolicy.FIRST_LOOP)
                .policyReset(ResetPolicy.BLOCK_LEFT)
                .policySpill(SpillPolicy.REALLOCATE)
                .policyAllocation(AllocationPolicy.OVERALLOCATE)
                .build();



        for( int i=0; i<10; i++ ){
            try(val ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(WS_ALL_LAYERS_ACT_CONFIG, "ws")){
//                System.out.println("STARTING: " + i);

                INDArray objectPresentMask = Nd4j.create(DataType.BOOL, 1,4,4);

                long[] shape = {1,3,2,4,4};
                INDArray noIntMask1 = Nd4j.createUninitialized(DataType.BOOL, shape, 'c');
                INDArray noIntMask2 = Nd4j.createUninitialized(DataType.BOOL, shape, 'c');

                noIntMask1 = Transforms.or(noIntMask1.get(all(), all(), point(0), all(), all()), noIntMask1.get(all(), all(), point(1), all(), all()) );    //Shape: [mb, b, H, W]. Values 1 if no intersection
                noIntMask2 = Transforms.or(noIntMask2.get(all(), all(), point(0), all(), all()), noIntMask2.get(all(), all(), point(1), all(), all()) );
                INDArray noIntMask = Transforms.or(noIntMask1, noIntMask2 );

                Nd4j.getExecutioner().commit();

                INDArray intMask = Transforms.not(noIntMask); //Values 0 if no intersection
                Nd4j.getExecutioner().commit();

                Broadcast.mul(intMask, objectPresentMask, intMask, 0, 2, 3);
                Nd4j.getExecutioner().commit();
//                System.out.println("DONE: " + i);
            }
        }
    }
 

@Test
public void testBroadcastMul_bool() {
    val mask = Nd4j.create(DataType.BOOL, 1, 3, 4, 4);
    val object = Nd4j.create(DataType.BOOL, 1, 4, 4);

    Broadcast.mul(mask, object, mask, 0, 2, 3);
    Nd4j.getExecutioner().commit();
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    if (epsilon.rank() != 3)
        throw new DL4JInvalidInputException("Got rank " + epsilon.rank()
                        + " array as epsilon for Subsampling1DLayer backprop with shape "
                        + Arrays.toString(epsilon.shape())
                        + ". Expected rank 3 array with shape [minibatchSize, features, length]. " + layerId());
    if(maskArray != null){
        INDArray maskOut = feedForwardMaskArray(maskArray, MaskState.Active, (int)epsilon.size(0)).getFirst();
        Preconditions.checkState(epsilon.size(0) == maskOut.size(0) && epsilon.size(2) == maskOut.size(1),
                "Activation gradients dimensions (0,2) and mask dimensions (0,1) don't match: Activation gradients %s, Mask %s",
                epsilon.shape(), maskOut.shape());
        Broadcast.mul(epsilon, maskOut, epsilon, 0, 2);
    }

    // add singleton fourth dimension to input and next layer's epsilon
    INDArray origInput = input;
    input = input.castTo(dataType).reshape(input.size(0), input.size(1), input.size(2), 1);
    epsilon = epsilon.reshape(epsilon.size(0), epsilon.size(1), epsilon.size(2), 1);

    // call 2D SubsamplingLayer's backpropGradient method
    Pair<Gradient, INDArray> gradientEpsNext = super.backpropGradient(epsilon, workspaceMgr);
    INDArray epsNext = gradientEpsNext.getSecond();

    // remove singleton fourth dimension from input and current epsilon
    input = origInput;
    epsNext = epsNext.reshape(epsNext.size(0), epsNext.size(1), epsNext.size(2));

    return new Pair<>(gradientEpsNext.getFirst(), epsNext);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (input.rank() != 3)
        throw new DL4JInvalidInputException("Got rank " + input.rank()
                        + " array as input to Subsampling1DLayer with shape " + Arrays.toString(input.shape())
                        + ". Expected rank 3 array with shape [minibatchSize, features, length]. " + layerId());

    // add singleton fourth dimension to input
    INDArray origInput = input;
    input = input.castTo(dataType).reshape(input.size(0), input.size(1), input.size(2), 1);

    // call 2D SubsamplingLayer's activate method
    INDArray acts = super.activate(training, workspaceMgr);

    // remove singleton fourth dimension from input and output activations
    input = origInput;
    acts = acts.reshape(acts.size(0), acts.size(1), acts.size(2));

    if(maskArray != null){
        INDArray maskOut = feedForwardMaskArray(maskArray, MaskState.Active, (int)acts.size(0)).getFirst();
        Preconditions.checkState(acts.size(0) == maskOut.size(0) && acts.size(2) == maskOut.size(1),
                "Activations dimensions (0,2) and mask dimensions (0,1) don't match: Activations %s, Mask %s",
                acts.shape(), maskOut.shape());
        Broadcast.mul(acts, maskOut, acts, 0, 2);
    }

    return acts;
}
 

@Override
public void apply(INDArray param){
    INDArray norm = param.norm2(dimensions);
    INDArray clipped = norm.unsafeDuplication();
    BooleanIndexing.replaceWhere(clipped, maxNorm, Conditions.greaterThan(maxNorm));
    norm.addi(epsilon);
    clipped.divi(norm);

    Broadcast.mul(param, clipped, param, getBroadcastDims(dimensions, param.rank()) );
}
 

@Override
public INDArray backprop(INDArray gradAtOutput, INDArray gradAtInput, int iteration, int epoch) {
    Preconditions.checkState(mask != null, "Cannot perform backprop: Dropout mask array is absent (already cleared?)");
    //Mask has values 0 or 1/p
    //dL/dIn = dL/dOut * dOut/dIn = dL/dOut * (0 if dropped, or 1/p otherwise)
    Broadcast.mul(gradAtOutput, mask, gradAtInput, 0, 1);
    mask = null;
    return gradAtInput;
}
 

public static INDArray activate(@NonNull INDArray boundingBoxPriors, @NonNull INDArray input, boolean nchw, LayerWorkspaceMgr layerWorkspaceMgr){
    if(!nchw)
        input = input.permute(0,3,1,2); //NHWC to NCHW

    long mb = input.size(0);
    long h = input.size(2);
    long w = input.size(3);
    long b = boundingBoxPriors.size(0);
    long c = input.size(1)/b-5;  //input.size(1) == b * (5 + C) -> C = (input.size(1)/b) - 5

    INDArray output = layerWorkspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), input.shape(), 'c');
    INDArray output5 = output.reshape('c', mb, b, 5+c, h, w);
    INDArray output4 = output;  //output.get(all(), interval(0,5*b), all(), all());
    INDArray input4 = input.dup('c');    //input.get(all(), interval(0,5*b), all(), all()).dup('c');
    INDArray input5 = input4.reshape('c', mb, b, 5+c, h, w);

    //X/Y center in grid: sigmoid
    INDArray predictedXYCenterGrid = input5.get(all(), all(), interval(0,2), all(), all());
    Transforms.sigmoid(predictedXYCenterGrid, false);

    //width/height: prior * exp(input)
    INDArray predictedWHPreExp = input5.get(all(), all(), interval(2,4), all(), all());
    INDArray predictedWH = Transforms.exp(predictedWHPreExp, false);
    Broadcast.mul(predictedWH, boundingBoxPriors.castTo(input.dataType()), predictedWH, 1, 2);  //Box priors: [b, 2]; predictedWH: [mb, b, 2, h, w]

    //Confidence - sigmoid
    INDArray predictedConf = input5.get(all(), all(), point(4), all(), all());   //Shape: [mb, B, H, W]
    Transforms.sigmoid(predictedConf, false);

    output4.assign(input4);

    //Softmax
    //TODO OPTIMIZE?
    INDArray inputClassesPreSoftmax = input5.get(all(), all(), interval(5, 5+c), all(), all());   //Shape: [minibatch, C, H, W]
    INDArray classPredictionsPreSoftmax2d = inputClassesPreSoftmax.permute(0,1,3,4,2) //[minibatch, b, c, h, w] To [mb, b, h, w, c]
            .dup('c').reshape('c', new long[]{mb*b*h*w, c});
    Transforms.softmax(classPredictionsPreSoftmax2d, false);
    INDArray postSoftmax5d = classPredictionsPreSoftmax2d.reshape('c', mb, b, h, w, c ).permute(0, 1, 4, 2, 3);

    INDArray outputClasses = output5.get(all(), all(), interval(5, 5+c), all(), all());   //Shape: [minibatch, C, H, W]
    outputClasses.assign(postSoftmax5d);

    if(!nchw)
        output = output.permute(0,2,3,1);       //NCHW to NHWC

    return output;
}
 

private static INDArray applyMask(INDArray input, INDArray maskArray, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(maskArray == null){
        return workspaceMgr.leverageTo(type, input);
    }
    switch (input.rank()){
        case 2:
            if(!maskArray.isColumnVectorOrScalar() || maskArray.size(0) != input.size(0)){
                throw new IllegalStateException("Expected column vector for mask with 2d input, with same size(0)" +
                        " as input. Got mask with shape: " + Arrays.toString(maskArray.shape()) +
                        ", input shape = " + Arrays.toString(input.shape()));
            }
            return workspaceMgr.leverageTo(type, input.mulColumnVector(maskArray));
        case 3:
            //Time series input, shape [Minibatch, size, tsLength], Expect rank 2 mask
            if(maskArray.rank() != 2 || input.size(0) != maskArray.size(0) || input.size(2) != maskArray.size(1)){
                throw new IllegalStateException("With 3d (time series) input with shape [minibatch, size, sequenceLength]=" +
                        Arrays.toString(input.shape()) + ", expected 2d mask array with shape [minibatch, sequenceLength]." +
                        " Got mask with shape: "+ Arrays.toString(maskArray.shape()));
            }
            INDArray fwd = workspaceMgr.createUninitialized(type, input.dataType(), input.shape(), 'f');
            Broadcast.mul(input, maskArray, fwd, 0, 2);
            return fwd;
        case 4:
            //CNN input. Expect column vector to be shape [mb,1,h,1], [mb,1,1,w], or [mb,1,h,w]
            int[] dimensions = new int[4];
            int count = 0;
            for(int i=0; i<4; i++ ){
                if(input.size(i) == maskArray.size(i)){
                    dimensions[count++] = i;
                }
            }
            if(count < 4){
                dimensions = Arrays.copyOfRange(dimensions, 0, count);
            }

            INDArray fwd2 = workspaceMgr.createUninitialized(type, input.dataType(), input.shape(), 'c');
            Broadcast.mul(input, maskArray, fwd2, dimensions);
            return fwd2;
        default:
            throw new RuntimeException("Expected rank 2 to 4 input. Got rank " + input.rank() + " with shape "
                    + Arrays.toString(input.shape()));
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    INDArray z = preOutput(true, workspaceMgr);
    INDArray delta = layerConf().getActivationFn().backprop(z, epsilon).getFirst(); //Shape: [mb, vector, seqLength]

    boolean ncw = layerConf().getOutputFormat() == RNNFormat.NCW;

    if (maskArray != null) {
        if(ncw){
            delta = Broadcast.mul(delta, maskArray, delta, 0, 2);
        } else {
            delta = Broadcast.mul(delta, maskArray, delta, 0, 1);
        }
    }

    int inputLength = layerConf().getInputLength();
    long numSamples = input.size(0);
    val nOut = layerConf().getNOut();

    if (delta.ordering() != 'c' || delta.isView() || !hasDefaultStridesForShape(delta)){
        delta = delta.dup('c');
    }

    if(ncw){
        delta = delta.permute(0, 2, 1);     //From [minibatch, nOut, length] to [minibatch, length, nOut]
    }

    delta = delta.reshape('c',inputLength * numSamples, nOut);

    INDArray weightGradients = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
    weightGradients.assign(0);

    if (!hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'f');

    INDArray indices = Nd4j.createFromArray(indexes);
    Nd4j.scatterUpdate(org.nd4j.linalg.api.ops.impl.scatter.ScatterUpdate.UpdateOp.ADD, weightGradients, indices, delta, WEIGHT_DIM);

    Gradient ret = new DefaultGradient();
    ret.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradients);

    if (hasBias()) {
        INDArray biasGradientsView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGradientsView, 0); //biasGradientView is initialized/zeroed first in sum op
        ret.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradientsView);
    }

    return new Pair<>(ret, null);
}