Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#max()

private INDArray activateHelperFullArray(INDArray inputArray, int[] poolDim) {
    switch (poolingType) {
        case MAX:
            return inputArray.max(poolDim);
        case AVG:
            return inputArray.mean(poolDim);
        case SUM:
            return inputArray.sum(poolDim);
        case PNORM:
            //P norm: https://arxiv.org/pdf/1311.1780.pdf
            //out = (1/N * sum( |in| ^ p) ) ^ (1/p)
            int pnorm = layerConf().getPnorm();

            INDArray abs = Transforms.abs(inputArray, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(poolDim);

            return Transforms.pow(pNorm, 1.0 / pnorm, false);
        default:
            throw new RuntimeException("Unknown or not supported pooling type: " + poolingType + " " + layerId());
    }
}
 

public SpTree(INDArray data, Collection<INDArray> indices, String similarityFunction) {
    this.indices = indices;
    this.N = data.rows();
    this.D = data.columns();
    this.similarityFunction = similarityFunction;
    data = data.dup();
    INDArray meanY = data.mean(0);
    INDArray minY = data.min(0);
    INDArray maxY = data.max(0);
    INDArray width = Nd4j.create(data.dataType(), meanY.shape());
    for (int i = 0; i < width.length(); i++) {
        width.putScalar(i, Math.max(maxY.getDouble(i) - meanY.getDouble(i),
                meanY.getDouble(i) - minY.getDouble(i)) + Nd4j.EPS_THRESHOLD);
    }

    try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
        init(null, data, meanY, width, indices, similarityFunction);
        fill(N);
    }
}
 

public Construct4dDataSet(int nExamples, int nChannels, int height, int width) {

            INDArray allImages = Nd4j.rand(new int[] {nExamples, nChannels, height, width});
            allImages.get(NDArrayIndex.all(), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all()).muli(100)
                            .addi(200);
            allImages.get(NDArrayIndex.all(), NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.all()).muli(0.001)
                            .subi(10);

            INDArray labels = Nd4j.linspace(1, nChannels, nChannels).reshape(nChannels, 1);
            sampleDataSet = new DataSet(allImages, labels);

            expectedMean = allImages.mean(0, 2, 3);
            expectedStd = allImages.std(0, 2, 3);

            expectedLabelMean = labels.mean(0);
            expectedLabelStd = labels.std(0);

            expectedMin = allImages.min(0, 2, 3);
            expectedMax = allImages.max(0, 2, 3);
        }
 

/**
 * Pass in a matrix
 * @param data
 */
public QuadTree(INDArray data) {
    INDArray meanY = data.mean(0);
    INDArray minY = data.min(0);
    INDArray maxY = data.max(0);
    init(data, meanY.getDouble(0), meanY.getDouble(1),
                    max(maxY.getDouble(0) - meanY.getDouble(0), meanY.getDouble(0) - minY.getDouble(0))
                                    + Nd4j.EPS_THRESHOLD,
                    max(maxY.getDouble(1) - meanY.getDouble(1), meanY.getDouble(1) - minY.getDouble(1))
                                    + Nd4j.EPS_THRESHOLD);
    fill();
}
 

/**
 * Scales the ndarray columns
 * to the given min/max values
 *
 * @param min the minimum number
 * @param max the max number
 */
public static void scaleMinMax(double min, double max, INDArray toScale) {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) X_scaled = X_std * (max - min) + min

    INDArray min2 = toScale.min(0);
    INDArray max2 = toScale.max(0);

    INDArray std = toScale.subRowVector(min2).diviRowVector(max2.sub(min2));

    INDArray scaled = std.mul(max - min).addi(min);
    toScale.assign(scaled);
}
 

/**
 * Divides each row by its max
 *
 * @param toScale the matrix to divide by its row maxes
 */
public static void scaleByMax(INDArray toScale) {
    INDArray scale = toScale.max(1);
    for (int i = 0; i < toScale.rows(); i++) {
        double scaleBy = scale.getDouble(i);
        toScale.putRow(i, toScale.getRow(i).divi(scaleBy));
    }
}
 

/**
 * Scales the ndarray columns
 * to the given min/max values
 *
 * @param min the minimum number
 * @param max the max number
 */
public static void scaleMinMax(double min, double max, INDArray toScale) {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) X_scaled = X_std * (max - min) + min

    INDArray min2 = toScale.min(0);
    INDArray max2 = toScale.max(0);

    INDArray std = toScale.subRowVector(min2).diviRowVector(max2.sub(min2));

    INDArray scaled = std.mul(max - min).addi(min);
    toScale.assign(scaled);
}
 

/**
 * Divides each row by its max
 *
 * @param toScale the matrix to divide by its row maxes
 */
public static void scaleByMax(INDArray toScale) {
    INDArray scale = toScale.max(1);
    for (int i = 0; i < toScale.rows(); i++) {
        double scaleBy = scale.getDouble(i);
        toScale.putRow(i, toScale.getRow(i).divi(scaleBy));
    }
}
 

/**
 * Add rows of data to the statistics
 *
 * @param data the matrix containing multiple rows of data to include
 * @param mask (optionally) the mask of the data, useful for e.g. time series
 */
public MinMaxStats.Builder add(@NonNull INDArray data, INDArray mask) {
    data = DataSetUtil.tailor2d(data, mask);
    if (data == null) {
        // Nothing to add. Either data is empty or completely masked. Just skip it, otherwise we will get
        // null pointer exceptions.
        return this;
    }

    INDArray tad = data.javaTensorAlongDimension(0, 0);
    INDArray batchMin = data.min(0);
    INDArray batchMax = data.max(0);
    if (!Arrays.equals(batchMin.shape(), batchMax.shape()))
        throw new IllegalStateException(
                        "Data min and max must be same shape. Likely a bug in the operation changing the input?");
    if (runningLower == null) {
        // First batch
        // Create copies because min and max are views to the same data set, which will cause problems with the
        // side effects of Transforms.min and Transforms.max
        runningLower = batchMin.dup();
        runningUpper = batchMax.dup();
    } else {
        // Update running bounds
        Transforms.min(runningLower, batchMin, false);
        Transforms.max(runningUpper, batchMax, false);
    }

    return this;
}
 

@Test
public void testPooling2D_Same() {
    int[] miniBatches = {1, 3, 5};
    int[] depths = {1, 3, 5};
    int[] inHeights = {5, 21};
    int[] inWidths = {5, 21};
    int[] strideH = {1, 2};
    int[] strideW = {1, 2};
    int[] sizeW = {1, 2, 3};
    int[] sizeH = {1, 2, 3};
    int[] padH = {0};
    int[] padW = {0};
    Pooling2D.Pooling2DType[] types = new Pooling2D.Pooling2DType[]{Pooling2D.Pooling2DType.AVG, Pooling2D.Pooling2DType.PNORM, Pooling2D.Pooling2DType.MAX,};

    int cnt = 0;

    for (Pooling2D.Pooling2DType type : types) {
        log.info("Trying pooling type: [{}]", type);
        for (int m : miniBatches) {
            for (int d : depths) {
                for (int h : inHeights) {
                    for (int w : inWidths) {
                        for (int sh : strideH) {
                            for (int sw : strideW) {
                                for (int kh : sizeH) {
                                    for (int kw : sizeW) {

                                        INDArray in = Nd4j.rand(new int[]{m, d, h, w});

                                        int[] outSize = getOutputSize(in, new int[]{kh, kw}, new int[]{sh, sw}, null, true);

                                        //Calculate padding for same mode:
                                        int pHTotal = (outSize[0]-1)*sh + kh - h;
                                        int pWTotal = (outSize[1]-1)*sw + kw - w;
                                        int padTop = pHTotal / 2;
                                        int padLeft = pWTotal / 2;

                                        INDArray col = Nd4j.create(new int[]{m, d, outSize[0], outSize[1], kh, kw}, 'c');
                                        INDArray col2 = col.permute(0, 1, 4, 5, 2, 3);
                                        //INDArray col = Nd4j.createUninitialized(new int[]{m, d, kh, kw, outSize[0], outSize[1]}, 'c');
                                        //INDArray col2 = col;

                                        Convolution.im2col(in, kh, kw, sh, sw, padTop, padLeft, true, col2);

                                        INDArray col2d = col.reshape('c', m * d * outSize[0] * outSize[1], kh * kw);

                                        INDArray output = Nd4j.create(m, d, outSize[0], outSize[1]);



                                        INDArray reduced = null;
                                        switch (type) {
                                            case PNORM:
                                                int pnorm = 3;

                                                Transforms.abs(col2d, false);
                                                Transforms.pow(col2d, pnorm, false);
                                                reduced = col2d.sum(1);
                                                Transforms.pow(reduced, (1.0 / pnorm), false);

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.PNORM, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        (double) pnorm, outSize[0], outSize[1], output);

                                                break;
                                            case MAX:
                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.MAX, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.max(1);
                                                break;
                                            case AVG:

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.AVG, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.mean(1);
                                                break;
                                        }

                                        reduced = reduced.reshape('c',m,d, outSize[0], outSize[1]);

                                        assertEquals("Failed opType: " + type, reduced, output);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
 

/**
 * This method returns the gradient of the cost function with respect to the
 * output from the previous layer.  For this cost function, the gradient
 * is derived from Bishop's paper "Mixture Density Networks" (1994) which
 * gives an elegant closed-form expression for the derivatives with respect
 * to each of the output components.
 * @param labels Labels to train on.
 * @param preOutput Output of neural network before applying the final activation function.
 * @param activationFn Activation function of output layer.
 * @param mask Mask to apply to gradients.
 * @return Gradient of cost function with respect to preOutput parameters.
 */
@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    long nSamples = labels.size(0);

    INDArray output = activationFn.getActivation(preOutput.dup(), false);

    MixtureDensityComponents mdc = extractComponents(output);

    INDArray gradient = Nd4j.zeros(nSamples, preOutput.columns());

    INDArray labelsMinusMu = labelsMinusMu(labels, mdc.mu);
    INDArray labelsMinusMuSquared = labelsMinusMu.mul(labelsMinusMu).sum(2);

    // This computes pi_i, see Bishop equation (30).
    // See http://www.plsyard.com/dealing-overflow-and-underflow-in-softmax-function/
    // this post for why we calculate the pi_i in this way.
    // With the exponential function here, we have to be very careful
    // about overflow/underflow considerations even with
    // fairly intermediate values.  Subtracting the max
    // here helps to ensure over/underflow does not happen here.
    // This isn't exactly a softmax because there's an 'alpha' coefficient
    // here, but the technique works, nonetheless.
    INDArray variance = mdc.sigma.mul(mdc.sigma);
    INDArray minustwovariance = variance.mul(2).negi();
    INDArray normalPart = mdc.alpha.div(Transforms.pow(mdc.sigma.mul(SQRT_TWO_PI), mLabelWidth));
    INDArray exponent = labelsMinusMuSquared.div(minustwovariance);
    INDArray exponentMax = exponent.max(1);
    exponent.subiColumnVector(exponentMax);
    INDArray pi = Transforms.exp(exponent).muli(normalPart);
    INDArray piDivisor = pi.sum(1);
    pi.diviColumnVector(piDivisor);

    // See Bishop equation (35)
    //INDArray dLdZAlpha = Nd4j.zeros(nSamples, nLabelsPerSample, mMixturesPerLabel); //mdc.alpha.sub(pi);
    INDArray dLdZAlpha = mdc.alpha.sub(pi);
    // See Bishop equation (38)
    INDArray dLdZSigma = (labelsMinusMuSquared.div(variance).subi(mLabelWidth)).muli(-1).muli(pi);
    // See Bishop equation (39)

    // This turned out to be way less efficient than
    // the simple 'for' loop here.
    //INDArray dLdZMu = pi
    //        .div(variance)
    //        .reshape(nSamples, mMixtures, 1)
    //        .repeat(2, mLabelWidth)
    //        .muli(labelsMinusMu)
    //        .negi()
    //        .reshape(nSamples, mMixtures * mLabelWidth);

    INDArray dLdZMu = Nd4j.create(nSamples, mMixtures, mLabelWidth);
    for (int k = 0; k < mLabelWidth; k++) {
        dLdZMu.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(k)},
                        labelsMinusMu.get(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(),
                                        NDArrayIndex.point(k)}).muli(pi).divi(variance).negi());
    }
    dLdZMu = dLdZMu.reshape(nSamples, mMixtures * mLabelWidth);

    // Place components of gradient into gradient holder.
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, mMixtures)}, dLdZAlpha);
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(mMixtures, mMixtures * 2)},
                    dLdZSigma);
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(),
                    NDArrayIndex.interval(mMixtures * 2, (mLabelWidth + 2) * mMixtures)}, dLdZMu);

    INDArray gradients = activationFn.backprop(preOutput, gradient).getFirst();

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Test
public void testNorm2_1() {
    INDArray array = Nd4j.rand(1769472, 9);

    INDArray max = array.max(1);
}
 

@Test
public void testPooling2D_Same() {
    int[] miniBatches = {1, 3, 5};
    int[] depths = {1, 3, 5};
    int[] inHeights = {5, 21};
    int[] inWidths = {5, 21};
    int[] strideH = {1, 2};
    int[] strideW = {1, 2};
    int[] sizeW = {1, 2, 3};
    int[] sizeH = {1, 2, 3};
    int[] padH = {0};
    int[] padW = {0};
    Pooling2D.Pooling2DType[] types = new Pooling2D.Pooling2DType[]{Pooling2D.Pooling2DType.PNORM, Pooling2D.Pooling2DType.AVG, Pooling2D.Pooling2DType.MAX};

    int cnt = 0;

    for (Pooling2D.Pooling2DType type : types) {
        log.info("Trying pooling type: [{}]", type);
        for (int m : miniBatches) {
            for (int d : depths) {
                for (int h : inHeights) {
                    for (int w : inWidths) {
                        for (int sh : strideH) {
                            for (int sw : strideW) {
                                for (int kh : sizeH) {
                                    for (int kw : sizeW) {

                                        INDArray in = Nd4j.linspace(1, (m * d * h * w), (m * d * h * w), Nd4j.defaultFloatingPointType()).reshape(new int[]{m, d, h, w});

                                        int[] outSize = getOutputSize(in, new int[]{kh, kw}, new int[]{sh, sw}, null, true);

                                        //Calculate padding for same mode:
                                        int pHTotal = (outSize[0]-1)*sh + kh - h;
                                        int pWTotal = (outSize[1]-1)*sw + kw - w;
                                        int padTop = pHTotal / 2;
                                        int padLeft = pWTotal / 2;

                                        INDArray col = Nd4j.create(new int[]{m, d, outSize[0], outSize[1], kh, kw}, 'c');
                                        INDArray col2 = col.permute(0, 1, 4, 5, 2, 3);
                                        //INDArray col = Nd4j.createUninitialized(new int[]{m, d, kH, kW, outSize[0], outSize[1]}, 'c');
                                        //INDArray col2 = col;

                                        Convolution.im2col(in, kh, kw, sh, sw, padTop, padLeft, true, col2);

                                        INDArray col2d = col.reshape('c', m * d * outSize[0] * outSize[1], kh * kw);

                                        INDArray output = Nd4j.create(m, d, outSize[0], outSize[1]);



                                        INDArray reduced = null;
                                        switch (type) {
                                            case PNORM:
                                                int pnorm = 3;

                                                Transforms.abs(col2d, false);
                                                Transforms.pow(col2d, pnorm, false);
                                                reduced = col2d.sum(1);
                                                Transforms.pow(reduced, (1.0 / pnorm), false);

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.PNORM, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        (double) pnorm, outSize[0], outSize[1], output);

                                                break;
                                            case MAX:
                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.MAX, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.max(1);
                                                break;
                                            case AVG:

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.AVG, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.mean(1);
                                                break;
                                        }

                                        reduced = reduced.reshape('c',m,d, outSize[0], outSize[1]).dup('c');

                                        assertEquals("Failed opType: " + type, reduced, output);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
 

public static INDArray maskedPoolingConvolution(PoolingType poolingType, INDArray toReduce, INDArray mask, int pnorm, DataType dataType) {
    if(mask.rank() != 4){
        //TODO BETTER ERROR MESSAGE EXPLAINING FORMAT
        //TODO ALSO HANDLE LEGACY FORMAT WITH WARNING WHERE POSSIBLE
        throw new IllegalStateException("Expected rank 4 mask array: Got array with shape " + Arrays.toString(mask.shape()));
    }

    mask = mask.castTo(dataType);   //no-op if already correct dtype

    // [minibatch, channels, h, w] data with a mask array of shape [minibatch, 1, X, Y]
    // where X=(1 or inH) and Y=(1 or inW)

    //General case: must be equal or 1 on each dimension
    int[] dimensions = new int[4];
    int count = 0;
    for(int i=0; i<4; i++ ){
        if(toReduce.size(i) == mask.size(i)){
            dimensions[count++] = i;
        }
    }
    if(count < 4){
        dimensions = Arrays.copyOfRange(dimensions, 0, count);
    }

    switch (poolingType) {
        case MAX:
            //TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
            INDArray negInfMask;
            if(mask.dataType() == DataType.BOOL){
                negInfMask = Transforms.not(mask).castTo(dataType);
            } else {
                negInfMask = mask.rsub(1.0);
            }
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, dimensions));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            return withInf.max(2, 3);
        case AVG:
        case SUM:
            INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, dimensions));

            INDArray summed = masked.sum(2, 3);
            if (poolingType == PoolingType.SUM) {
                return summed;
            }
            INDArray maskCounts = mask.sum(1,2,3);
            summed.diviColumnVector(maskCounts);
            return summed;

        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, dimensions));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(2, 3);

            return Transforms.pow(pNorm, 1.0 / pnorm);
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
 

@Test
public void testNorm2_1() throws Exception {
    INDArray array = Nd4j.rand(1769472, 9);

    INDArray max = array.max(1);
}
 

@Test
public void testMax0() throws Exception {
    INDArray array1 = Nd4j.linspace(1, 76800,76800).reshape(256, 300);



    long time1 = System.currentTimeMillis();
    INDArray array = array1.max(0);
    long time2 = System.currentTimeMillis();

    System.out.println("Array1 shapeInfo: " + array1.shapeInfoDataBuffer());
    System.out.println("Result shapeInfo: " + array.shapeInfoDataBuffer());

    System.out.println("Time elapsed: "+ (time2 - time1) );

    assertEquals(300, array.length());

    for (int x = 0; x < 300; x++) {
        assertEquals("Failed on x: " + x, 76800 - (array1.columns() - x) + 1 , array.getFloat(x), 0.01f);
    }
}
 

/**
 * This method returns the gradient of the cost function with respect to the
 * output from the previous layer.  For this cost function, the gradient
 * is derived from Bishop's paper "Mixture Density Networks" (1994) which
 * gives an elegant closed-form expression for the derivatives with respect
 * to each of the output components.
 * @param labels Labels to train on.
 * @param preOutput Output of neural network before applying the final activation function.
 * @param activationFn Activation function of output layer.
 * @param mask Mask to apply to gradients.
 * @return Gradient of cost function with respect to preOutput parameters.
 */
@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    long nSamples = labels.size(0);

    INDArray output = activationFn.getActivation(preOutput.dup(), false);

    MixtureDensityComponents mdc = extractComponents(output);

    INDArray gradient = Nd4j.zeros(nSamples, preOutput.columns());

    INDArray labelsMinusMu = labelsMinusMu(labels, mdc.mu);
    INDArray labelsMinusMuSquared = labelsMinusMu.mul(labelsMinusMu).sum(2);

    // This computes pi_i, see Bishop equation (30).
    // See http://www.plsyard.com/dealing-overflow-and-underflow-in-softmax-function/
    // this post for why we calculate the pi_i in this way.
    // With the exponential function here, we have to be very careful
    // about overflow/underflow considerations even with
    // fairly intermediate values.  Subtracting the max
    // here helps to ensure over/underflow does not happen here.
    // This isn't exactly a softmax because there's an 'alpha' coefficient
    // here, but the technique works, nonetheless.
    INDArray variance = mdc.sigma.mul(mdc.sigma);
    INDArray minustwovariance = variance.mul(2).negi();
    INDArray normalPart = mdc.alpha.div(Transforms.pow(mdc.sigma.mul(SQRT_TWO_PI), mLabelWidth));
    INDArray exponent = labelsMinusMuSquared.div(minustwovariance);
    INDArray exponentMax = exponent.max(1);
    exponent.subiColumnVector(exponentMax);
    INDArray pi = Transforms.exp(exponent).muli(normalPart);
    INDArray piDivisor = pi.sum(true,1);
    pi.diviColumnVector(piDivisor);

    // See Bishop equation (35)
    //INDArray dLdZAlpha = Nd4j.zeros(nSamples, nLabelsPerSample, mMixturesPerLabel); //mdc.alpha.sub(pi);
    INDArray dLdZAlpha = mdc.alpha.sub(pi);
    // See Bishop equation (38)
    INDArray dLdZSigma = (labelsMinusMuSquared.div(variance).subi(mLabelWidth)).muli(-1).muli(pi);
    // See Bishop equation (39)

    // This turned out to be way less efficient than
    // the simple 'for' loop here.
    //INDArray dLdZMu = pi
    //        .div(variance)
    //        .reshape(nSamples, mMixtures, 1)
    //        .repeat(2, mLabelWidth)
    //        .muli(labelsMinusMu)
    //        .negi()
    //        .reshape(nSamples, mMixtures * mLabelWidth);

    INDArray dLdZMu = Nd4j.create(nSamples, mMixtures, mLabelWidth);
    for (int k = 0; k < mLabelWidth; k++) {
        dLdZMu.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(k)},
                        labelsMinusMu.get(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(),
                                        NDArrayIndex.point(k)}).muli(pi).divi(variance).negi());
    }
    dLdZMu = dLdZMu.reshape(nSamples, mMixtures * mLabelWidth);

    // Place components of gradient into gradient holder.
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(0, mMixtures)}, dLdZAlpha);
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.interval(mMixtures, mMixtures * 2)},
                    dLdZSigma);
    gradient.put(new INDArrayIndex[] {NDArrayIndex.all(),
                    NDArrayIndex.interval(mMixtures * 2, (mLabelWidth + 2) * mMixtures)}, dLdZMu);

    INDArray gradients = activationFn.backprop(preOutput, gradient).getFirst();

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public INDArray op(INDArray neighbourhoodFeatures, INDArray nodeFeature) {
    return neighbourhoodFeatures.max(0);
}
 

public static INDArray maskedPoolingTimeSeries(PoolingType poolingType, INDArray toReduce, INDArray mask,
                int pnorm, DataType dataType) {
    if (toReduce.rank() != 3) {
        throw new IllegalArgumentException("Expect rank 3 array: got " + toReduce.rank());
    }
    if (mask.rank() != 2) {
        throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
    }

    toReduce = toReduce.castTo(dataType);
    mask = mask.castTo(dataType);

    //Sum pooling: easy. Multiply by mask, then sum as normal
    //Average pooling: as above, but do a broadcast element-wise divi by mask.sum(1)
    //Max pooling: set to -inf if mask is 0, then do max as normal

    switch (poolingType) {
        case MAX:
            INDArray negInfMask = mask.castTo(dataType).rsub(1.0);
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, 0, 2));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            return withInf.max(2);
        case AVG:
        case SUM:
            INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, 0, 2));
            INDArray summed = masked.sum(2);
            if (poolingType == PoolingType.SUM) {
                return summed;
            }

            INDArray maskCounts = mask.sum(1);
            summed.diviColumnVector(maskCounts);
            return summed;
        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, 0, 2));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = abs.sum(2);

            return Transforms.pow(pNorm, 1.0 / pnorm);
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
 

protected void op(INDArray x, INDArray y, int i) {
    // broadcast along row & column
    INDArray row = Nd4j.ones(64);
    INDArray column = Nd4j.ones(1024, 1);

    x.addiRowVector(row);
    x.addiColumnVector(column);

    // casual scalar
    x.addi(i * 2);

    // reduction along all dimensions
    float sum = x.sumNumber().floatValue();

    // index reduction
    Nd4j.getExecutioner().exec(new IMax(x), Integer.MAX_VALUE);

    // casual transform
    Nd4j.getExecutioner().exec(new Sqrt(x, x));

    //  dup
    INDArray x1 = x.dup(x.ordering());
    INDArray x2 = x.dup(x.ordering());
    INDArray x3 = x.dup('c');
    INDArray x4 = x.dup('f');


    // vstack && hstack
    INDArray vstack = Nd4j.vstack(x, x1, x2, x3, x4);

    INDArray hstack = Nd4j.hstack(x, x1, x2, x3, x4);

    // reduce3 call
    Nd4j.getExecutioner().exec(new ManhattanDistance(x, x2));


    // flatten call
    INDArray flat = Nd4j.toFlattened(x, x1, x2, x3, x4);


    // reduction along dimension: row & column
    INDArray max_0 = x.max(0);
    INDArray max_1 = x.max(1);


    // index reduction along dimension: row & column
    INDArray imax_0 = Nd4j.argMax(x, 0);
    INDArray imax_1 = Nd4j.argMax(x, 1);


    // logisoftmax, softmax & softmax derivative
    Nd4j.getExecutioner().exec(new OldSoftMax(x));
    Nd4j.getExecutioner().exec(new SoftMaxDerivative(x));
    Nd4j.getExecutioner().exec(new LogSoftMax(x));


    // BooleanIndexing
    BooleanIndexing.replaceWhere(x, 5f, Conditions.lessThan(8f));

    // assing on view
    BooleanIndexing.assignIf(x, x1, Conditions.greaterThan(-1000000000f));

    // std var along all dimensions
    float std = x.stdNumber().floatValue();

    // std var along row & col
    INDArray xStd_0 = x.std(0);
    INDArray xStd_1 = x.std(1);

    // blas call
    float dot = (float) Nd4j.getBlasWrapper().dot(x, x1);

    // mmul
    for (boolean tA : paramsA) {
        for (boolean tB : paramsB) {

            INDArray xT = tA ? x.dup() : x.dup().transpose();
            INDArray yT = tB ? y.dup() : y.dup().transpose();

            Nd4j.gemm(xT, yT, tA, tB);
        }
    }

    // specially for views, checking here without dup and rollover
    Nd4j.gemm(x, y, false, false);

    log.debug("Iteration passed: " + i);
}