Java Code Examples for org.nd4j.linalg.primitives.Pair#getFirst()

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray previous, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); // Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);
    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, new long[] { w.size(0), delta.size(0) }, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();

    // Normally we would clear weightNoiseParams here - but we want to reuse them
    // for forward + backward + score
    // So this is instead done in MultiLayerNetwork/CompGraph backprop methods

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

@Test
public void testDupAndDupWithOrder() {
    List<Pair<INDArray, String>> testInputs = NDArrayCreationUtil.getAllTestMatricesWithShape(4, 5, 123);
    int count = 0;
    for (Pair<INDArray, String> pair : testInputs) {
        String msg = pair.getSecond();
        INDArray in = pair.getFirst();
        System.out.println("Count " + count);
        INDArray dup = in.dup();
        INDArray dupc = in.dup('c');
        INDArray dupf = in.dup('f');

        assertEquals(msg, in, dup);
        assertEquals(msg, dup.ordering(), (char) Nd4j.order());
        assertEquals(msg, dupc.ordering(), 'c');
        assertEquals(msg, dupf.ordering(), 'f');
        assertEquals(msg, in, dupc);
        assertEquals(msg, in, dupf);
        count++;
    }
}
 

private Pair<INDArray[], INDArray[]> convertFeatures(INDArray[] features, INDArray[] masks,
		List<SubsetDetails> subsetDetails, int minExamples, Map<String, List<List<Writable>>> nextRRVals,
		int longestFeaturesInput[], int[][] lengthFeaturesInputs) {
	boolean hasMasks = false;
	int i = 0;

	for (SubsetDetails d : subsetDetails) {
		if (nextRRVals.containsKey(d.readerName)) {
			List<List<Writable>> list = nextRRVals.get(d.readerName);
				Pair<INDArray, INDArray> p = paddingFeatures(list, minExamples, longestFeaturesInput[i], d, lengthFeaturesInputs[i]);
				features[i] = p.getFirst();
				masks[i] = p.getSecond();
				if (masks[i] != null)
					hasMasks = true;
		}
		i++;
	}

	return new Pair<>(features, hasMasks ? masks : null);
}
 

@Test
public void testBroadcastingGenerated() {
    int[][] broadcastShape = NDArrayCreationUtil.getRandomBroadCastShape(7, 6, 10);
    List<List<Pair<INDArray, String>>> broadCastList = new ArrayList<>(broadcastShape.length);
    for (int[] shape : broadcastShape) {
        List<Pair<INDArray, String>> arrShape = NDArrayCreationUtil.get6dPermutedWithShape(7, shape);
        broadCastList.add(arrShape);
        broadCastList.add(NDArrayCreationUtil.get6dReshapedWithShape(7, shape));
        broadCastList.add(NDArrayCreationUtil.getAll6dTestArraysWithShape(7, shape));
    }

    for (List<Pair<INDArray, String>> b : broadCastList) {
        for (Pair<INDArray, String> val : b) {
            INDArray inputArrBroadcast = val.getFirst();
            val destShape = NDArrayCreationUtil.broadcastToShape(inputArrBroadcast.shape(), 7);
            INDArray output = inputArrBroadcast
                            .broadcast(NDArrayCreationUtil.broadcastToShape(inputArrBroadcast.shape(), 7));
            assertArrayEquals(destShape, output.shape());
        }
    }



}
 

/**
 * Merge all of the features arrays into one minibatch.
 *
 * @param featuresToMerge     features to merge. Note that first index is the input array (example) index, the second
 *                            index is the input array.
 *                            Thus to merge 10 examples with 3 input arrays each, featuresToMerge will be indexed
 *                            like featuresToMerge[0..9][0..2]
 * @param featureMasksToMerge May be null. If non-null: feature masks to merge
 * @return Merged features, and feature masks. Note that feature masks may be added automatically, if required - even
 * if no feature masks were present originally
 */
public static Pair<INDArray[], INDArray[]> mergeFeatures(@NonNull INDArray[][] featuresToMerge, INDArray[][] featureMasksToMerge) {
    int nInArrs = featuresToMerge[0].length;
    INDArray[] outF = new INDArray[nInArrs];
    INDArray[] outM = null;

    for (int i = 0; i < nInArrs; i++) {
        Pair<INDArray, INDArray> p = mergeFeatures(featuresToMerge, featureMasksToMerge, i);
        outF[i] = p.getFirst();
        if (p.getSecond() != null) {
            if (outM == null) {
                outM = new INDArray[nInArrs];
            }
            outM[i] = p.getSecond();
        }
    }
    return new Pair<>(outF, outM);
}
 

/**
 * Register a set of classes (Layer, GraphVertex, InputPreProcessor, IActivation, ILossFunction, ReconstructionDistribution
 * ONLY) for JSON deserialization, with custom names.<br>
 * Using this method directly should never be required (instead: use {@link #registerLegacyCustomClassesForJSON(Class[])}
 * but is added in case it is required in non-standard circumstances.
 */
public static void registerLegacyCustomClassesForJSON(List<Pair<String,Class>> classes){
    for(Pair<String,Class> p : classes){
        String s = p.getFirst();
        Class c = p.getRight();
        //Check if it's a valid class to register...
        boolean found = false;
        for( Class<?> c2 : REGISTERABLE_CUSTOM_CLASSES){
            if(c2.isAssignableFrom(c)){
                Map<String,String> map = LegacyMappingHelper.legacyMappingForClass(c2);
                map.put(p.getFirst(), p.getSecond().getName());
                found = true;
            }
        }

        if(!found){
            throw new IllegalArgumentException("Cannot register class for legacy JSON deserialization: class " +
                    c.getName() + " is not a subtype of classes " + REGISTERABLE_CUSTOM_CLASSES);
        }
    }
}
 

public static void compareArrays(int rank, char ordering) {
    List<Pair<INDArray, String>> all = NDArrayCreationUtil.getTestMatricesWithVaryingShapes(rank,ordering);
    Iterator<Pair<INDArray,String>> iter = all.iterator();
    while (iter.hasNext()) {
        Pair<INDArray,String> currentPair = iter.next();
        INDArray origArray = currentPair.getFirst();
        //adding elements outside the bounds where print switches to scientific notation
        origArray.tensorAlongDimension(0,0).muli(0).addi(100000);
        origArray.putScalar(0,10001.1234);
        log.info("\nChecking shape ..." + currentPair.getSecond());
        log.info("\n"+ origArray.dup('c').toString());
        Nd4j.writeTxt(origArray, "someArr.txt");
        INDArray readBack = Nd4j.readTxt("someArr.txt");
        assertEquals("\nNot equal on shape " + ArrayUtils.toString(origArray.shape()), origArray, readBack);
        try {
            Files.delete(Paths.get("someArr.txt"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
 

@Test
public void testDelayedTAD1() throws Exception {
    TADManager tadManager = new DeviceTADManager();

    INDArray array = Nd4j.create(128, 256);

    Pair<DataBuffer, DataBuffer> tadBuffers = tadManager.getTADOnlyShapeInfo(array, new int[]{0});

    DataBuffer tadBuffer = tadBuffers.getFirst();
    DataBuffer offBuffer = tadBuffers.getSecond();

    AllocationPoint pointTad = AtomicAllocator.getInstance().getAllocationPoint(tadBuffer);
    AllocationPoint pointOff = AtomicAllocator.getInstance().getAllocationPoint(offBuffer);

    assertEquals(AllocationStatus.CONSTANT, pointTad.getAllocationStatus());
    assertEquals(AllocationStatus.DEVICE, pointOff.getAllocationStatus());
}
 

@Test
public void testInverseComparison() {

    List<Pair<INDArray, String>> list = NDArrayCreationUtil.getAllTestMatricesWithShape(10, 10, 12345);

    for (Pair<INDArray, String> p : list) {
        INDArray orig = p.getFirst();
        orig.assign(Nd4j.rand(orig.shape()));
        INDArray inverse = InvertMatrix.invert(orig, false);
        RealMatrix rm = CheckUtil.convertToApacheMatrix(orig);
        RealMatrix rmInverse = new LUDecomposition(rm).getSolver().getInverse();

        INDArray expected = CheckUtil.convertFromApacheMatrix(rmInverse);
        assertTrue(p.getSecond(), CheckUtil.checkEntries(expected, inverse, 1e-3, 1e-4));
    }
}
 

@Test
public void testToOffsetZeroCopy() {
    List<Pair<INDArray, String>> testInputs = NDArrayCreationUtil.getAllTestMatricesWithShape(4, 5, 123);

    for (Pair<INDArray, String> pair : testInputs) {
        String msg = pair.getSecond();
        INDArray in = pair.getFirst();
        INDArray dup = Shape.toOffsetZeroCopy(in);
        INDArray dupc = Shape.toOffsetZeroCopy(in, 'c');
        INDArray dupf = Shape.toOffsetZeroCopy(in, 'f');
        INDArray dupany = Shape.toOffsetZeroCopyAnyOrder(in);

        assertEquals(msg, in, dup);
        assertEquals(msg, in, dupc);
        assertEquals(msg, in, dupf);
        assertEquals(msg, dupc.ordering(), 'c');
        assertEquals(msg, dupf.ordering(), 'f');
        assertEquals(msg, in, dupany);

        assertEquals(dup.offset(), 0);
        assertEquals(dupc.offset(), 0);
        assertEquals(dupf.offset(), 0);
        assertEquals(dupany.offset(), 0);
        assertEquals(dup.length(), dup.data().length());
        assertEquals(dupc.length(), dupc.data().length());
        assertEquals(dupf.length(), dupf.data().length());
        assertEquals(dupany.length(), dupany.data().length());
    }
}
 

@Test
public void checkSliceofSlice() {
    /*
        Issue 1: Slice of slice with c order and f order views are not equal

        Comment out assert and run then -> Issue 2: Index out of bound exception with certain shapes when accessing elements with getDouble() in f order
        (looks like problem is when rank-1==1) eg. 1,2,1 and 2,2,1
     */
    int[] ranksToCheck = new int[]{2, 3, 4, 5};
    for (int rank = 0; rank < ranksToCheck.length; rank++) {
        log.info("\nRunning through rank " + ranksToCheck[rank]);
        List<Pair<INDArray, String>> allF = NDArrayCreationUtil.getTestMatricesWithVaryingShapes(ranksToCheck[rank], 'f');
        Iterator<Pair<INDArray, String>> iter = allF.iterator();
        while (iter.hasNext()) {
            Pair<INDArray, String> currentPair = iter.next();
            INDArray origArrayF = currentPair.getFirst();
            INDArray sameArrayC = origArrayF.dup('c');
            log.info("\nLooping through slices for shape " + currentPair.getSecond());
            log.info("\nOriginal array:\n" + origArrayF);
            INDArray viewF = origArrayF.slice(0);
            INDArray viewC = sameArrayC.slice(0);
            log.info("\nSlice 0, C order:\n" + viewC.toString());
            log.info("\nSlice 0, F order:\n" + viewF.toString());
            for (int i = 0; i < viewF.slices(); i++) {
                //assertEquals(viewF.slice(i),viewC.slice(i));
                for (int j = 0; j < viewF.slice(i).length(); j++) {
                    //if (j>0) break;
                    log.info("\nC order slice " + i + ", element 0 :" + viewC.slice(i).getDouble(j)); //C order is fine
                    log.info("\nF order slice " + i + ", element 0 :" + viewF.slice(i).getDouble(j)); //throws index out of bound err on F order
                }
            }
        }
    }
}
 

@Test
public void testExpandDims(){
    final List<Pair<INDArray, String>> testMatricesC = NDArrayCreationUtil.getAllTestMatricesWithShape('c', 3, 5, 0xDEAD);
    final List<Pair<INDArray, String>> testMatricesF = NDArrayCreationUtil.getAllTestMatricesWithShape('f', 7, 11, 0xBEEF);

    final ArrayList<Pair<INDArray, String>> testMatrices = new ArrayList<>(testMatricesC);
    testMatrices.addAll(testMatricesF);

    for (Pair<INDArray, String> testMatrixPair : testMatrices) {
        final String recreation = testMatrixPair.getSecond();
        final INDArray testMatrix = testMatrixPair.getFirst();
        final char ordering = testMatrix.ordering();
        val shape = testMatrix.shape();
        final int rank = testMatrix.rank();
        for (int i = -rank; i <= rank; i++) {
            final INDArray expanded = Nd4j.expandDims(testMatrix, i);

            final String message = "Expanding in Dimension " + i + "; Shape before expanding: " + Arrays.toString(shape) + " "+ordering+" Order; Shape after expanding: " + Arrays.toString(expanded.shape()) +  " "+expanded.ordering()+"; Input Created via: " + recreation;

            assertEquals(message, 1, expanded.shape()[i < 0 ? i + rank : i]);
            assertEquals(message, testMatrix.ravel(), expanded.ravel());
            assertEquals(message, ordering,  expanded.ordering());

            testMatrix.assign(Nd4j.rand(shape));
            assertEquals(message, testMatrix.ravel(), expanded.ravel());
        }
    }
}
 

@Override
public Pair<DataBuffer, DataBuffer> getTADOnlyShapeInfo(INDArray array, int[] dimension) {
    /*
        so, we check, if we have things cached.
        If we don't - we just create new TAD shape, and push it to constant memory
    */
    if (dimension != null && dimension.length > 1)
        Arrays.sort(dimension);

    Integer deviceId = AtomicAllocator.getInstance().getDeviceId();

    //log.info("Requested TAD for device [{}], dimensions: [{}]", deviceId, Arrays.toString(dimension));

    //extract the dimensions and shape buffer for comparison
    TadDescriptor descriptor = new TadDescriptor(array, dimension);

    if (!tadCache.get(deviceId).containsKey(descriptor)) {
        log.trace("Creating new TAD...");
        //create the TAD with the shape information and corresponding offsets
        //note that we use native code to get access to the shape information.
        Pair<DataBuffer, DataBuffer> buffers = super.getTADOnlyShapeInfo(array, dimension);
        /**
         * Store the buffers in constant memory.
         * The main implementation of this is cuda right now.
         *
         * Explanation from: http://cuda-programming.blogspot.jp/2013/01/what-is-constant-memory-in-cuda.html
         * The CUDA language makes available another kind of memory known as constant memory. As the opName may indicate, we use constant memory for data that will not change over the course of a kernel execution.
        
         Why Constant Memory?
        
         NVIDIA hardware provides 64KB of constant memory that
         it treats differently than it treats standard global memory. In some situations,
         using constant memory rather than global memory will reduce the required memory bandwidth.
        
         NOTE HERE FOR US: We use 48kb of it using these methods.
        
         Note also that we use the {@link AtomicAllocator} which is the cuda memory manager
         for moving the current host space data buffer to constant memory.
        
         We do this for device access to shape information.
         */
        if (buffers.getFirst() != array.shapeInfoDataBuffer())
            AtomicAllocator.getInstance().moveToConstant(buffers.getFirst());
        /**
         * @see {@link org.nd4j.jita.constant.ProtectedCudaConstantHandler}
         */
        if (buffers.getSecond() != null)
            AtomicAllocator.getInstance().moveToConstant(buffers.getSecond());

        // so, at this point we have buffer valid on host side.
        // And we just need to replace DevicePointer with constant pointer
        tadCache.get(deviceId).put(descriptor, buffers);

        bytes.addAndGet((buffers.getFirst().length() * 4));

        if (buffers.getSecond() != null)
            bytes.addAndGet(buffers.getSecond().length() * 8);

        log.trace("Using TAD from cache...");
    }

    return tadCache.get(deviceId).get(descriptor);
}
 

@Override
public INDArray tensorAlongDimension(int index, int... dimension) {
    if (dimension == null || dimension.length == 0)
        throw new IllegalArgumentException("Invalid input: dimensions not specified (null or length 0)");

    if (dimension.length >= rank()  || dimension.length == 1 && dimension[0] == Integer.MAX_VALUE)
        return this;
    for (int i = 0; i < dimension.length; i++)
        if (dimension[i] < 0)
            dimension[i] += rank();

    //dedup
    if (dimension.length > 1)
        dimension = Ints.toArray(new ArrayList<>(new TreeSet<>(Ints.asList(dimension))));

    if (dimension.length > 1) {
        Arrays.sort(dimension);
    }

    long tads = tensorssAlongDimension(dimension);
    if (index >= tads)
        throw new IllegalArgumentException("Illegal index " + index + " out of tads " + tads);


    if (dimension.length == 1) {
        if (dimension[0] == 0 && isColumnVector()) {
            return this.transpose();
        } else if (dimension[0] == 1 && isRowVector()) {
            return this;
        }
    }

    Pair<DataBuffer, DataBuffer> tadInfo =
            Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(this, dimension);
    DataBuffer shapeInfo = tadInfo.getFirst();
    val shape = Shape.shape(shapeInfo);
    val stride = Shape.stride(shapeInfo).asLong();
    long offset = offset() + tadInfo.getSecond().getLong(index);
    INDArray toTad = Nd4j.create(data(), shape, stride, offset);
    BaseNDArray baseNDArray = (BaseNDArray) toTad;

    //preserve immutability
    char newOrder = Shape.getOrder(shape, stride, 1);

    int ews = baseNDArray.shapeInfoDataBuffer().getInt(baseNDArray.shapeInfoDataBuffer().length() - 2);

    //TAD always calls permute. Permute EWS is always -1. This is not true
    // for row vector shapes though.
    if (!Shape.isRowVectorShape(baseNDArray.shapeInfoDataBuffer()))
        ews = -1;

    // we create new shapeInfo with possibly new ews & order
    /**
     * NOTE HERE THAT ZERO IS PRESET FOR THE OFFSET AND SHOULD STAY LIKE THAT.
     * Zero is preset for caching purposes.
     * We don't actually use the offset defined in the
     * shape info data buffer.
     * We calculate and cache the offsets separately.
     *
     */
    baseNDArray.setShapeInformation(
            Nd4j.getShapeInfoProvider().createShapeInformation(shape, stride, 0, ews, newOrder));

    return toTad;
}
 

@Override
public Long apply(Pair<Long, Long> input) {
    return input.getFirst() + input.getSecond();
}
 

@Test
    @Ignore
    public void testMaxPoolBackprop(){
        Nd4j.getRandom().setSeed(12345);

        for( int i=0; i<5; i++ ) {

            int[] inputShape = {1, 1, 4, 3};

            int[] kernel = {2, 2};
            int[] strides = {1, 1};
            int[] pad = {0, 0};
            int[] dilation = {1, 1};        //TODO non 1-1 dilation
            boolean same = true;


            String fn = "maxpool2d_bp";
            int nIArgs = 11;

            int[] a = new int[nIArgs];
            a[0] = kernel[0];
            a[1] = kernel[1];
            a[2] = strides[0];
            a[3] = strides[1];
            a[4] = pad[0];
            a[5] = pad[1];
            a[6] = dilation[0];
            a[7] = dilation[1];
            a[8] = same ? 1 : 0;
            //a[9]: Not used with max pooling
            a[10] = 0;  //For NCHW

            List<Pair<INDArray, String>> inputs = NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, inputShape);

            for(Pair<INDArray,String> pIn : inputs){
                INDArray input = pIn.getFirst();
                int[] outShapeHW = getOutputSize(input, kernel, strides, pad, same);
                List<Pair<INDArray, String>> eps = NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, inputShape[0], inputShape[1], outShapeHW[0], outShapeHW[1]);
                for(Pair<INDArray,String> pEps : eps){
                    INDArray epsilon = pEps.getFirst();
                    INDArray epsNext = Nd4j.create(inputShape, 'c');

                    //Runs fine with dups:
//                    input = input.dup('c');
                    epsilon = epsilon.dup('c');

                    DynamicCustomOp op = DynamicCustomOp.builder(fn)
                            .addInputs(input, epsilon)
                            .addOutputs(epsNext)
                            .addIntegerArguments(a)
                            .build();

                    Nd4j.getExecutioner().exec(op);

                    INDArray expEpsNext = expGradMaxPoolBackPropSame(input, epsilon, kernel, strides, same);

                    String msg = "input=" + pIn.getSecond() + ", eps=" + pEps.getSecond();
                    assertEquals(msg, expEpsNext, epsNext);
                }
            }
        }
    }
 

@Test
public void testBufferToIntShapeStrideMethods() {
    //Specifically: Shape.shape(IntBuffer), Shape.shape(DataBuffer)
    //.isRowVectorShape(DataBuffer), .isRowVectorShape(IntBuffer)
    //Shape.size(DataBuffer,int), Shape.size(IntBuffer,int)
    //Also: Shape.stride(IntBuffer), Shape.stride(DataBuffer)
    //Shape.stride(DataBuffer,int), Shape.stride(IntBuffer,int)

    List<List<Pair<INDArray, String>>> lists = new ArrayList<>();
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 4, 12345));
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(1, 4, 12345));
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 1, 12345));
    lists.add(NDArrayCreationUtil.getAll3dTestArraysWithShape(12345, 3, 4, 5));
    lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, 3, 4, 5, 6));
    lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, 3, 1, 5, 1));
    lists.add(NDArrayCreationUtil.getAll5dTestArraysWithShape(12345, 3, 4, 5, 6, 7));
    lists.add(NDArrayCreationUtil.getAll6dTestArraysWithShape(12345, 3, 4, 5, 6, 7, 8));

    val shapes = new long[][] {{3, 4}, {1, 4}, {3, 1}, {3, 4, 5}, {3, 4, 5, 6}, {3, 1, 5, 1}, {3, 4, 5, 6, 7},
                    {3, 4, 5, 6, 7, 8}};

    for (int i = 0; i < shapes.length; i++) {
        List<Pair<INDArray, String>> list = lists.get(i);
        val shape = shapes[i];

        for (Pair<INDArray, String> p : list) {
            INDArray arr = p.getFirst();

            assertArrayEquals(shape, arr.shape());

            val thisStride = arr.stride();

            val ib = arr.shapeInfo();
            DataBuffer db = arr.shapeInfoDataBuffer();

            //Check shape calculation
            assertEquals(shape.length, Shape.rank(ib));
            assertEquals(shape.length, Shape.rank(db));

            assertArrayEquals(shape, Shape.shape(ib));
            assertArrayEquals(shape, Shape.shape(db));

            for (int j = 0; j < shape.length; j++) {
                assertEquals(shape[j], Shape.size(ib, j));
                assertEquals(shape[j], Shape.size(db, j));

                assertEquals(thisStride[j], Shape.stride(ib, j));
                assertEquals(thisStride[j], Shape.stride(db, j));
            }

            //Check base offset
            assertEquals(Shape.offset(ib), Shape.offset(db));

            //Check offset calculation:
            NdIndexIterator iter = new NdIndexIterator(shape);
            while (iter.hasNext()) {
                val next = iter.next();
                long offset1 = Shape.getOffset(ib, next);

                assertEquals(offset1, Shape.getOffset(db, next));

                switch (shape.length) {
                    case 2:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1]));
                        break;
                    case 3:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2]));
                        break;
                    case 4:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2], next[3]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2], next[3]));
                        break;
                    case 5:
                    case 6:
                        //No 5 and 6d getOffset overloads
                        break;
                    default:
                        throw new RuntimeException();
                }
            }
        }
    }
}
 

protected void setShapeInformation(Pair<DataBuffer, long[]> shapeInfo) {
    this.shapeInformation = shapeInfo.getFirst();
    this.javaShapeInformation = shapeInfo.getSecond();
}
 

public MultiDataSet nextMultiDataSet(Map<String, List<List<Writable>>> nextRRVals,
		List<RecordMetaDataComposableMap> nextMetas) {
	int minExamples = Integer.MAX_VALUE;
	for (List<List<Writable>> exampleData : nextRRVals.values()) {
		minExamples = Math.min(minExamples, exampleData.size());
	}

	// Support for VastText
	int[] longestRecord = new int[inputs.size()];
	int recordLength;

	//This allows to aling text vectors of different lenghts in the batch
	int recordPos = inputs.get(0).subsetStart;
	int inputPos = 0;
	List<List<Writable>> list;
	int[][] inputsLength = new int[inputs.size()][minExamples];
	for (Entry<String, List<List<Writable>>> entry : nextRRVals.entrySet()) {

		list = entry.getValue();
		for (int i = 0; i < list.size() && i < minExamples; i++) {
			// Analysis of Records
			recordLength = (int) ((NDArrayWritable) list.get(i).get(recordPos)).length();
			inputsLength[inputPos][i] = Math.max(inputsLength[inputPos][i], recordLength);
			longestRecord[inputPos] = Math.max(longestRecord[inputPos], recordLength);
		}
		inputPos++;
	}

	if (minExamples == Integer.MAX_VALUE)
		throw new RuntimeException("Error occurred during data set generation: no readers?"); // Should never happen

	Pair<INDArray[], INDArray[]> features = convertFeatures(new INDArray[inputs.size()],
			new INDArray[inputs.size()], inputs, minExamples, nextRRVals, longestRecord, inputsLength);

	Pair<INDArray[], INDArray[]> labels = convertLabels(new INDArray[outputs.size()],
			new INDArray[outputs.size()], outputs, minExamples, nextRRVals);

	MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(features.getFirst(), labels.getFirst(),
			features.getSecond(), labels.getSecond());
	if (collectMetaData) {
		mds.setExampleMetaData(nextMetas);
	}
	if (preProcessor != null)
		preProcessor.preProcess(mds);
	return mds;
}
 

private void setShapeInformation(Pair<DataBuffer, long[]> shapeInfo) {
    this.shapeInformation = shapeInfo.getFirst();
    this.javaShapeInformation = shapeInfo.getSecond();
}