Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#dup()

@Test
public void testCBOWGradientNoOp1() throws Exception {
    INDArray syn0 = Nd4j.create(10, 10).assign(0.01f);
    INDArray syn1 = Nd4j.create(10, 10).assign(0.02f);
    INDArray syn1Neg = Nd4j.ones(10, 10).assign(0.03f);
    INDArray expTable = Nd4j.create(10000).assign(0.5f);
    INDArray table = Nd4j.create(100000);

    double lr = 0.025;

    INDArray expSyn0 = syn0.dup();
    INDArray expSyn1 = syn1.dup();
    INDArray expSyn1Neg = syn1Neg.dup();

    AggregateCBOW op = new AggregateCBOW(syn0, syn1, syn1Neg, expTable, table, 0, new int[] {}, new int[] {},
                    new int[] {}, 0, 0, 10, lr, 2L, 10);

    Nd4j.getExecutioner().exec(op);

    assertEquals(expSyn0, syn0);
    assertEquals(expSyn1, syn1);
    assertEquals(expSyn1Neg, syn1Neg);
}
 

@Test
public void testReplaceWhereArray() {
    for (Condition c : new Condition[]{Conditions.lessThan(0.5), Conditions.greaterThan(0.5), Conditions.equals(0.5)}) {

        INDArray inArr = Nd4j.rand(3, 4);
        INDArray inArr2 = Nd4j.valueArrayOf(3, 4, 10);
        SameDiff sd = SameDiff.create();
        SDVariable in = sd.var("in", inArr);
        SDVariable in2 = sd.var("in2", inArr2);
        SDVariable where = sd.replaceWhere(in, in2, c);

        INDArray exp = inArr.dup();
        BooleanIndexing.replaceWhere(exp, inArr2, c);

        SDVariable loss = where.std(true);

        TestCase tc = new TestCase(sd);

        String err = OpValidation.validate(tc);
        assertNull(err);
    }
}
 

@Test
public void testDupSkipDecompression3() {
    INDArray array = Nd4j.linspace(1, 100, 2500, DataType.FLOAT);

    INDArray compressed = Nd4j.getCompressor().compress(array, "GZIP");

    INDArray newArray = compressed.dup('f');
    assertFalse(newArray.isCompressed());

    Nd4j.getCompressor().decompressi(compressed);
    //        Nd4j.getCompressor().decompressi(newArray);

    assertEquals(array, compressed);
    assertEquals(array, newArray);
    assertEquals('f', newArray.ordering());
    assertEquals('c', compressed.ordering());
}
 

@Override
public double negLogProbability(INDArray x, INDArray preOutDistributionParams, boolean average) {
    //p(x) = lambda * exp( -lambda * x)
    //logp(x) = log(lambda) - lambda * x = gamma - lambda * x

    INDArray gamma = preOutDistributionParams.dup();
    activationFn.getActivation(gamma, false);

    INDArray lambda = Transforms.exp(gamma, true);
    double negLogProbSum = -lambda.muli(x).rsubi(gamma).sumNumber().doubleValue();
    if (average) {
        return negLogProbSum / x.size(0);
    } else {
        return negLogProbSum;
    }

}
 

@Test
public void testFactorSVDTransposed() {
    int m = 4;
    int n = 13;

    double f[] = new double[] {7, 1, 11, 11, 7, 11, 3, 1, 2, 21, 1, 11, 10, 26, 29, 56, 31, 52, 55, 71, 31, 54, 47,
                    40, 66, 68, 6, 15, 8, 8, 6, 9, 17, 22, 18, 4, 23, 9, 8, 60, 52, 20, 47, 33, 22, 6, 44, 22, 26,
                    34, 12, 12};

    INDArray A = Nd4j.create(f, new long[] {m, n}, 'f');

    INDArray A1 = A.dup('f');
    INDArray factor = org.nd4j.linalg.dimensionalityreduction.PCA.pca_factor(A1, 3, true);
    A1 = A.subiRowVector(A.mean(0));

    INDArray reduced = A1.mmul(factor);
    INDArray reconstructed = reduced.mmul(factor.transpose());
    INDArray diff = reconstructed.sub(A1);
    for (int i = 0; i < m * n; i++) {
        assertEquals("Reconstructed matrix is very different from the original.", 0.0, diff.getDouble(i), 1.0);
    }
}
 

@Test
public void testCreate() {
    INDArray array0 = Nd4j.create(new double[]{1,2,3,4,5,6,7,8,9});
    INDArray array0_1 = array0.dup();
    INDArray array0_2 = array0.dup();
    INDArray array0_3 = array0.dup();
    INDArray array1 = Nd4j.create(new double[]{1,2,3,4,5,6,7,8,9}, new int[]{3, 3}, 'c');
    INDArray array1_1 = array1.dup();
    INDArray array1_2 = array1.dup();
    INDArray array1_3 = array1.dup();
    INDArray array2 = Nd4j.create(new double[]{1,2,3,4,5,6,7,8,9,10,11,12, 13,14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}, new int[]{3, 3, 3}, 'c');
    INDArray array2_1 = array2.dup();
    INDArray array2_2 = array2.dup();
    INDArray array2_3 = array2.dup();

    assertEquals(2, array1.rank());
    assertEquals(3, array2.rank());

    INDArray array2D = Nd4j.vstack(array0, array0_1, array0_2, array0_3);
    INDArray array3D = Nd4j.vstack(array1, array1_1, array1_2, array1_3);
    INDArray array4D = Nd4j.vstack(array2, array2_1, array2_2, array2_3);

    log.info("Output Array2D rank: {}", array2D.rank());
    log.info("Output Array3D rank: {}", array3D.rank());
    log.info("Output Array4D rank: {}", array4D.rank());
}
 

/**
 * Equation (6) from the Gatys et all paper: https://arxiv.org/pdf/1508.06576.pdf
 * This is the derivative of the style error for a single layer w.r.t. the
 * combo image features at that layer.
 *
 * @param styleGramFeatures Intermediate activations of one layer for style input
 * @param comboFeatures     Intermediate activations of one layer for combo image input
 * @return Derivative of style error matrix for the layer w.r.t. combo image
 */
private INDArray derivativeLossStyleInLayer(INDArray styleGramFeatures, INDArray comboFeatures) {

    comboFeatures = comboFeatures.dup();
    double N = comboFeatures.shape()[0];
    double M = comboFeatures.shape()[1] * comboFeatures.shape()[2];

    double styleWeight = 1.0 / ((N * N) * (M * M));
    // Corresponds to G^l in equation (6)
    INDArray contentGram = gramMatrix(comboFeatures);
    // G^l - A^l
    INDArray diff = contentGram.sub(styleGramFeatures);
    // (F^l)^T * (G^l - A^l)
    INDArray trans = flatten(comboFeatures).transpose();
    INDArray product = trans.mmul(diff);
    // (1/(N^2 * M^2)) * ((F^l)^T * (G^l - A^l))
    INDArray posResult = product.muli(styleWeight);
    // This multiplication assures that the result is 0 when the value from F^l < 0, but is still (1/(N^2 * M^2)) * ((F^l)^T * (G^l - A^l)) otherwise
    return posResult.muli(ensurePositive(trans));
}
 

@Test
public void testEmbedding(){
    INDArray z1 = Nd4j.randn(2000, 400);
    INDArray z2 = z1.dup();
    INDArray result = Transforms.allEuclideanDistances(z1, z2, 1);

    RandomProjection rp = new RandomProjection(0.5);
    INDArray zp = rp.project(z1);
    INDArray zp2 = zp.dup();
    INDArray projRes = Transforms.allEuclideanDistances(zp, zp2, 1);

    // check that the automatically tuned values for the density respect the
    // contract for eps: pairwise distances are preserved according to the
    // Johnson-Lindenstrauss lemma
    INDArray ratios = projRes.div(result);

    for (int i = 0; i < ratios.length(); i++){
        double val = ratios.getDouble(i);
        // this avoids the NaNs we get along the diagonal
        if (val == val) {
            assertTrue(ratios.getDouble(i) < 1.5);
        }
    }

}
 

@Test
public void testDelayedDup1() throws Exception {
    INDArray array = Nd4j.linspace(1,1000, 1000).reshape(10, 10, 10);

    AllocationPoint pointShape = AtomicAllocator.getInstance().getAllocationPoint(array.shapeInfoDataBuffer());
    AllocationPoint pointArray = AtomicAllocator.getInstance().getAllocationPoint(array);

    assertEquals(AllocationStatus.HOST, pointArray.getAllocationStatus());
    assertEquals(AllocationStatus.HOST, pointShape.getAllocationStatus());

    float sum = array.sumNumber().floatValue();

    pointShape = AtomicAllocator.getInstance().getAllocationPoint(array.shapeInfoDataBuffer());
    pointArray = AtomicAllocator.getInstance().getAllocationPoint(array);

    assertEquals(AllocationStatus.DEVICE, pointArray.getAllocationStatus());
    assertEquals(AllocationStatus.CONSTANT, pointShape.getAllocationStatus());

    INDArray dup = array.dup();

    AllocationPoint dupShape = AtomicAllocator.getInstance().getAllocationPoint(dup.shapeInfoDataBuffer());
    AllocationPoint dupArray = AtomicAllocator.getInstance().getAllocationPoint(dup);

    assertEquals(AllocationStatus.DEVICE, dupArray.getAllocationStatus());
    assertEquals(AllocationStatus.CONSTANT, dupShape.getAllocationStatus());
}
 

@Test
public void testEvalAdd() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Nd4j.linspace(1, 4, 4);
    INDArray yArr = arr.dup();
    SDVariable x = sameDiff.var("x", arr);
    SDVariable y = sameDiff.var("y", yArr);

    SDVariable sigmoid = x.mul(y);
    INDArray assertion = arr.mul(arr);
    Map<String, INDArray> vars = new HashMap<>();
    vars.put("x", arr);
    vars.put("y", yArr);
    INDArray eval = sameDiff.output(vars, Collections.singletonList(sigmoid.name())).get(sigmoid.name());
    assertEquals(assertion, eval);
}
 

@Test
public void testScalarMinMax1() {
    INDArray x = Nd4j.create(new double[] {1, 3, 5, 7});
    INDArray xCopy = x.dup();
    INDArray exp1 = Nd4j.create(new double[] {1, 3, 5, 7});
    INDArray exp2 = Nd4j.create(new double[] {1e-5, 1e-5, 1e-5, 1e-5});

    INDArray z1 = Transforms.max(x, Nd4j.EPS_THRESHOLD, true);
    INDArray z2 = Transforms.min(x, Nd4j.EPS_THRESHOLD, true);

    assertEquals(exp1, z1);
    assertEquals(exp2, z2);
    // Assert that x was not modified
    assertEquals(x, xCopy);

    INDArray exp3 = Nd4j.create(new double[] {10, 10, 10, 10});
    Transforms.max(x, 10, false);
    assertEquals(x, exp3);

    Transforms.min(x, Nd4j.EPS_THRESHOLD, false);
    assertEquals(x, exp2);
}
 

@Test
public void testDupSkipDecompression2() {
    INDArray array = Nd4j.linspace(1, 100, 2500, DataType.FLOAT);

    INDArray compressed = Nd4j.getCompressor().compress(array, "GZIP");

    INDArray newArray = compressed.dup('c');
    assertTrue(newArray.isCompressed());

    Nd4j.getCompressor().decompressi(compressed);
    Nd4j.getCompressor().decompressi(newArray);

    assertEquals(array, compressed);
    assertEquals(array, newArray);
}
 

private INDArray calcLogProbArray(INDArray x, INDArray preOutDistributionParams) {
    x = x.castTo(preOutDistributionParams.dataType());
    INDArray output = preOutDistributionParams.dup();
    activationFn.getActivation(output, false);

    INDArray logOutput = Transforms.log(output, true);
    INDArray log1SubOut = Transforms.log(output.rsubi(1.0), false);

    //For numerical stability: if output = 0, then log(output) == -infinity
    //then x * log(output) = NaN, but lim(x->0, output->0)[ x * log(output) ] == 0
    // therefore: want 0*log(0) = 0, NOT 0*log(0) = NaN by default
    BooleanIndexing.replaceWhere(logOutput, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0
    BooleanIndexing.replaceWhere(log1SubOut, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0
    return logOutput.muli(x).addi(x.rsub(1.0).muli(log1SubOut));
}
 

/**
 * Reshape time series mask arrays. This should match the assumptions (f order, etc) in RnnOutputLayer
 * @param timeSeriesMask    Mask array to reshape to a column vector
 * @return                  Mask array as a column vector
 */
public static INDArray reshapeTimeSeriesMaskToVector(INDArray timeSeriesMask) {
    if (timeSeriesMask.rank() != 2)
        throw new IllegalArgumentException("Cannot reshape mask: rank is not 2");

    if (timeSeriesMask.ordering() != 'f')
        timeSeriesMask = timeSeriesMask.dup('f');

    return timeSeriesMask.reshape('f', timeSeriesMask.length(), 1);
}
 

private Box(INDArray toDup) {
    data = toDup.dup();
}
 

@Test
    @Ignore
    public void testMaxPoolBackprop(){
        Nd4j.getRandom().setSeed(12345);

        for( int i=0; i<5; i++ ) {

            int[] inputShape = {1, 1, 4, 3};

            int[] kernel = {2, 2};
            int[] strides = {1, 1};
            int[] pad = {0, 0};
            int[] dilation = {1, 1};        //TODO non 1-1 dilation
            boolean same = true;


            String fn = "maxpool2d_bp";
            int nIArgs = 11;

            int[] a = new int[nIArgs];
            a[0] = kernel[0];
            a[1] = kernel[1];
            a[2] = strides[0];
            a[3] = strides[1];
            a[4] = pad[0];
            a[5] = pad[1];
            a[6] = dilation[0];
            a[7] = dilation[1];
            a[8] = same ? 1 : 0;
            //a[9]: Not used with max pooling
            a[10] = 0;  //For NCHW

            List<Pair<INDArray, String>> inputs = NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, inputShape, Nd4j.defaultFloatingPointType());

            for(Pair<INDArray,String> pIn : inputs){
                INDArray input = pIn.getFirst();
                int[] outShapeHW = getOutputSize(input, kernel, strides, pad, same);
                List<Pair<INDArray, String>> eps = NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, new int[]{inputShape[0], inputShape[1], outShapeHW[0], outShapeHW[1]}, Nd4j.defaultFloatingPointType());
                for(Pair<INDArray,String> pEps : eps){
                    INDArray epsilon = pEps.getFirst();
                    INDArray epsNext = Nd4j.create(inputShape, 'c');

                    //Runs fine with dups:
//                    input = input.dup('c');
                    epsilon = epsilon.dup('c');

                    DynamicCustomOp op = DynamicCustomOp.builder(fn)
                            .addInputs(input, epsilon)
                            .addOutputs(epsNext)
                            .addIntegerArguments(a)
                            .build();

                    Nd4j.getExecutioner().execAndReturn(op);

                    INDArray expEpsNext = expGradMaxPoolBackPropSame(input, epsilon, kernel, strides, same);

                    String msg = "input=" + pIn.getSecond() + ", eps=" + pEps.getSecond();
                    assertEquals(msg, expEpsNext, epsNext);
                }
            }
        }
    }
 

public static INDArray maskedPoolingEpsilonCnn(PoolingType poolingType, INDArray input, INDArray mask,
                INDArray epsilon2d, int pnorm, DataType dataType) {

    // [minibatch, channels, h=1, w=X] or [minibatch, channels, h=X, w=1] data
    // with a mask array of shape [minibatch, X]

    //If masking along height: broadcast dimensions are [0,2]
    //If masking along width: broadcast dimensions are [0,3]

    mask = mask.castTo(dataType);   //No-op if correct type

    //General case: must be equal or 1 on each dimension
    int[] dimensions = new int[4];
    int count = 0;
    for(int i=0; i<4; i++ ){
        if(input.size(i) == mask.size(i)){
            dimensions[count++] = i;
        }
    }
    if(count < 4){
        dimensions = Arrays.copyOfRange(dimensions, 0, count);
    }

    switch (poolingType) {
        case MAX:
            //TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
            INDArray negInfMask;
            if(mask.dataType() == DataType.BOOL){
                negInfMask = Transforms.not(mask).castTo(dataType);
            } else {
                negInfMask = mask.rsub(1.0);
            }
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));

            INDArray withInf = Nd4j.createUninitialized(dataType, input.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(input, negInfMask, withInf, dimensions));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op

            INDArray isMax = Nd4j.exec(new IsMax(withInf, withInf.ulike(), 2, 3))[0];

            return Nd4j.getExecutioner().exec(new BroadcastMulOp(isMax, epsilon2d, isMax, 0, 1));
        case AVG:
        case SUM:
            //if out = sum(in,dims) then dL/dIn = dL/dOut -> duplicate to each step and mask
            //if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
            //With masking: N differs for different time series

            INDArray out = Nd4j.createUninitialized(dataType, input.shape(), 'f');

            //Broadcast copy op, then divide and mask to 0 as appropriate
            Nd4j.getExecutioner().exec(new BroadcastCopyOp(out, epsilon2d, out, 0, 1));
            Nd4j.getExecutioner().exec(new BroadcastMulOp(out, mask, out, dimensions));

            if (poolingType == PoolingType.SUM) {
                return out;
            }

            //Note that with CNNs, current design is restricted to [minibatch, channels, 1, W] ot [minibatch, channels, H, 1]
            INDArray nEachTimeSeries = mask.sum(1,2,3); //[minibatchSize,tsLength] -> [minibatchSize,1]
            Nd4j.getExecutioner().exec(new BroadcastDivOp(out, nEachTimeSeries, out, 0));

            return out;

        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(dataType, input.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(input, mask, masked2, dimensions));

            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = Transforms.pow(abs.sum(2, 3), 1.0 / pnorm);

            INDArray numerator;
            if (pnorm == 2) {
                numerator = input.dup();
            } else {
                INDArray absp2 = Transforms.pow(Transforms.abs(input, true), pnorm - 2, false);
                numerator = input.mul(absp2);
            }

            INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
            denom.rdivi(epsilon2d);
            Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, 0, 1));
            Nd4j.getExecutioner().exec(new BroadcastMulOp(numerator, mask, numerator, dimensions)); //Apply mask

            return numerator;
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);

    }
}
 

/**
 * Solve using the preconditioned conjugate gradient method. The implementation is based on:
 *
 * <dt><a id="BARR1994">Barret et al. (1994)</a></dt>
 * <dd>R. Barrett, M. Berry, T. F. Chan, J. Demmel, J. M. Donato, J. Dongarra,
 * V. Eijkhout, R. Pozo, C. Romine and H. Van der Vorst,
 * <a href="http://www.netlib.org/linalg/html_templates/Templates.html"><em>
 * Templates for the Solution of Linear Systems: Building Blocks for Iterative
 * Methods</em></a>, SIAM</dd>
 *
 * If {@link #performDetailedChecks} is true, the (required) positive-definiteness condition
 * on the linear operator is checked during each iteration (with no computation cost) and an
 * exception is thrown if the condition is violated.
 *
 * The final solution is contained in the output {@link SubroutineSignal}, along with the norm
 * of the residual vector, exit status, etc; see {@link #generateConjugateGradientsSubroutineSignal}
 *
 * @param x0 initial guess
 * @return an instance of {@link SubroutineSignal} containing the solution and residual error
 * @throws IllegalArgumentException for bad initial guess or illegal linear operators
 * @throws NonPositiveDefiniteOperatorException if the linear operator is deemed to be non-positive-definite
 */
public SubroutineSignal solveUsingPreconditionedConjugateGradient(@Nonnull final INDArray x0) throws IllegalArgumentException {
    Utils.nonNull(x0, "The initial guess must be non-null");
    Utils.validateArg(x0.length() == linop.getRowDimension(), "The initial guess has wrong dimensions");
    Utils.validateArg(linop.getRowDimension() == linop.getColumnDimension(), "The CG routine only works for square" +
            " linear operators");

    /* initialize */
    int iter = 0;
    final double rmax = relTol * normFunc.apply(b);
    final INDArray x = x0.dup();
    final INDArray p = x.dup();
    final INDArray q = linop.operate(p);
    final INDArray r = b.sub(q);
    final INDArray z;
    z = precond == null ? r : Nd4j.create(r.shape());
    double rnorm = normFunc.apply(r);
    if (rnorm <= rmax) {
        return generateConjugateGradientsSubroutineSignal(x, iter, rnorm, rmax, ExitStatus.SUCCESS_REL_TOL);
    }

    double rhoPrev = 0;
    while (true) {
        /* z_{i-1} = M r_{i-1} */
        if (precond != null) {
            z.assign(precond.operate(r));
        }
        /* rho_{i-1} = r_{i-1}^T z_{i-1} */
        final double rhoNext = innerProductFunc.apply(r, z);
        if (performDetailedChecks && rhoNext <= 0) {
            throw new NonPositiveDefiniteOperatorException();
        }
        if (iter == 0) {
            /* p_1 = z_0 */
            p.assign(z);
        } else {
            /* p_{i} = z_{i-1}  + (\rho_{i-1}/\rho_{i-2}) p_{i-1} */
            p.muli(rhoNext / rhoPrev).addi(z);
        }
        /* q_{i} = A p_{i} */
        q.assign(linop.operate(p));
        final double pq = innerProductFunc.apply(p, q);
        if (performDetailedChecks && pq <= 0) {
            throw new NonPositiveDefiniteOperatorException();
        }
        /* \alpha_{i} = \rho_{i-1}/(p_{i}^T q_{i}^T) */
        final double alpha = rhoNext / pq;
        /* x_{i} = x_{i-1}  + \alpha_{i} p_{i} */
        x.addi(p.mul(alpha));
        /* r_{i} = r_{i-1}  - \alpha_{i} q_{i} */
        r.addi(q.mul(-alpha));
        rhoPrev = rhoNext;
        rnorm = normFunc.apply(r);
        if (rnorm <= rmax) {
            return generateConjugateGradientsSubroutineSignal(x, iter, rnorm, rmax, ExitStatus.SUCCESS_REL_TOL);
        }
        if (rnorm <= absTol) {
            return generateConjugateGradientsSubroutineSignal(x, iter, rnorm, rmax, ExitStatus.SUCCESS_ABS_TOL);
        }
        if (++iter >= maxIters) {
            return generateConjugateGradientsSubroutineSignal(x, iter, rnorm, rmax, ExitStatus.FAIL_MAX_ITERS);
        }
    }
}
 

public int ssyev( char _jobz, char _uplo, int N, INDArray A, INDArray R ) {

	int status = -1 ;

	int jobz = _jobz == 'V' ? CUSOLVER_EIG_MODE_VECTOR : CUSOLVER_EIG_MODE_NOVECTOR ;
	int uplo = _uplo == 'L' ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER ;

        if (Nd4j.dataType() != DataBuffer.Type.FLOAT)
            log.warn("FLOAT ssyev called in DOUBLE environment");

        INDArray a = A;

        if (A.ordering() == 'c')
            a = A.dup('f');

        // FIXME: int cast
	int M = (int) A.rows() ;

        if (Nd4j.getExecutioner() instanceof GridExecutioner)
            ((GridExecutioner) Nd4j.getExecutioner()).flushQueue();

        // Get context for current thread
        CudaContext ctx = (CudaContext) allocator.getDeviceContext().getContext();

        // setup the solver handles for cuSolver calls
        cusolverDnHandle_t handle = ctx.getSolverHandle();
        cusolverDnContext solverDn = new cusolverDnContext(handle);

        // synchronized on the solver
        synchronized (handle) {
            status = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getOldStream()));
            if( status == 0 ) {
		    // transfer the INDArray into GPU memory
		    CublasPointer xAPointer = new CublasPointer(a, ctx);
		    CublasPointer xRPointer = new CublasPointer(R, ctx);

		    // this output - indicates how much memory we'll need for the real operation
		    DataBuffer worksizeBuffer = Nd4j.getDataBufferFactory().createInt(1);
		    status = cusolverDnSsyevd_bufferSize (
				solverDn, jobz, uplo, M, 
				(FloatPointer) xAPointer.getDevicePointer(), M,
				(FloatPointer) xRPointer.getDevicePointer(),
				(IntPointer)worksizeBuffer.addressPointer() ) ;

		    if (status == CUSOLVER_STATUS_SUCCESS) {
			    int worksize = worksizeBuffer.getInt(0);

			    // allocate memory for the workspace, the non-converging row buffer and a return code
			    Pointer workspace = new Workspace(worksize * Nd4j.sizeOfDataType());

			    INDArray INFO = Nd4j.createArrayFromShapeBuffer(Nd4j.getDataBufferFactory().createInt(1),
		  	    Nd4j.getShapeInfoProvider().createShapeInformation(new int[] {1, 1}));


			    // Do the actual decomp
			    status = cusolverDnSsyevd(solverDn, jobz, uplo, M, 
					(FloatPointer) xAPointer.getDevicePointer(), M,
					(FloatPointer) xRPointer.getDevicePointer(), 
					new CudaPointer(workspace).asFloatPointer(), worksize,
					new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer());

			    allocator.registerAction(ctx, INFO);
			    if( status == 0 ) status = INFO.getInt(0) ;
		    }
		}
        }
	if( status == 0 ) {
		allocator.registerAction(ctx, R);
		allocator.registerAction(ctx, a);

		if (a != A)
		    A.assign(a);
	}
	return status ;
    }
 

@Test
    public void testCompGraphUpdaterBlocks(){
        //Check that setting learning rate results in correct rearrangement of updater state within updater blocks
        //https://github.com/deeplearning4j/deeplearning4j/issues/6809#issuecomment-463892644

        double lr = 1e-3;
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .weightInit(WeightInit.XAVIER)
                .updater(new Adam(lr))
                .graphBuilder()
                .backpropType(BackpropType.Standard)
                .addInputs("in")
                .setOutputs("out")
                .addLayer("0",new DenseLayer.Builder().nIn(5).nOut(3).build(),"in")
                .addLayer("1",new DenseLayer.Builder().nIn(3).nOut(2).build(),"0")
                .addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(2).nOut(1)
                        .activation(Activation.SIGMOID).build(),"1")
                .build();

        ComputationGraph cg = new ComputationGraph(conf);
        cg.init();

        INDArray in = Nd4j.rand(1, 5);
        INDArray lbl = Nd4j.rand(1,1);

        cg.fit(new DataSet(in, lbl));

        INDArray viewArray = cg.getUpdater().getUpdaterStateViewArray();
        INDArray viewArrayCopy = viewArray.dup();
        //Initially updater view array is set out like:
        //[m0w, m0b, m1w, m1b, m2w, m2b][v0w, v0b, v1w, v1b, v2w, v2b]
        long soFar = 0;
        INDArray m0w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+5*3)).assign(0);    //m0w
        soFar += 5*3;
        INDArray m0b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3)).assign(1);    //m0b
        soFar += 3;
        INDArray m1w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3*2)).assign(2);    //m1w
        soFar += 3*2;
        INDArray m1b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2)).assign(3);    //m1b
        soFar += 2;
        INDArray m2w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2*1)).assign(4);    //m2w
        soFar += 2*1;
        INDArray m2b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+1)).assign(5);    //m2b
        soFar += 1;

        INDArray v0w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+5*3)).assign(6);    //v0w
        soFar += 5*3;
        INDArray v0b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3)).assign(7);    //v0b
        soFar += 3;
        INDArray v1w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3*2)).assign(8);    //v1w
        soFar += 3*2;
        INDArray v1b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2)).assign(9);    //v1b
        soFar += 2;
        INDArray v2w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2*1)).assign(10);    //v2w
        soFar += 2*1;
        INDArray v2b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+1)).assign(11);    //v2b
        soFar += 1;


        cg.setLearningRate("0", 0.0);

        //Expect new updater state to look like:
        //[m0w, m0b][v0w,v0b], [m1w, m1b, m2w, m2b][v1w, v1b, v2w, v2b]
        INDArray exp = Nd4j.concat(1, m0w, m0b, v0w, v0b,
                m1w, m1b, m2w, m2b, v1w, v1b, v2w, v2b);

        INDArray act = cg.getUpdater().getUpdaterStateViewArray();
//        System.out.println(exp);
//        System.out.println(act);

        assertEquals(exp, act);

        //And set layer 1 LR:
        cg.setLearningRate("1", 0.2);
        exp = Nd4j.concat(1, m0w, m0b, v0w, v0b,
                m1w, m1b, v1w, v1b,
                m2w, m2b, v2w, v2b);
        assertEquals(exp, cg.getUpdater().getStateViewArray());


        //Set all back to original LR and check again:
        cg.setLearningRate("1", lr);
        cg.setLearningRate("0", lr);

        exp = Nd4j.concat(1, m0w, m0b, m1w, m1b, m2w, m2b, v0w, v0b, v1w, v1b, v2w, v2b);
        assertEquals(exp, cg.getUpdater().getStateViewArray());


        //Finally, training sanity check (if things are wrong, we get -ve values in adam V, which causes NaNs)
        cg.getUpdater().getStateViewArray().assign(viewArrayCopy);
        cg.setLearningRate("0", 0.0);

        Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.NAN_PANIC);
        cg.fit(new DataSet(in, lbl));
        Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC);
    }