Java Code Examples for org.nd4j.common.primitives.Pair#getSecond()

/**
 * Collect the data (List of datapoints containing tokens and an associated label.
 *
 * @param num Requested number of data points
 * @return List of datapoints
 */
protected List<Datum> collectData(int num) {
  LabeledSentenceProvider sentenceProvider = getSentenceProvider();
  String unknownWordSentinel = getUnknownWordSentinel();
  List<Datum> data = new ArrayList<>(num);
  for (int i = data.size(); i < num && sentenceProvider.hasNext(); i++) {
    Pair<String, String> p = sentenceProvider.nextSentence();
    String sentence = p.getFirst();
    String label = p.getSecond();
    List<String> tokens = tokenizeSentence(sentence);

    // Skip stopwords
    tokens = tokens.stream()
        .filter(s -> !stopwords.isStopword(s))
        .collect(Collectors.toList());

    // If tokens are empty, add at least the UNKNOWN_WORD_SENTINEL
    if (tokens.isEmpty()) {
      tokens.add(unknownWordSentinel);
    }

    data.add(new Datum(tokens, label));
  }
  return data;
}
 

private INDArray getLastStep(INDArray in, LayerWorkspaceMgr workspaceMgr, ArrayType arrayType){
        if(in.rank() != 3){
            throw new IllegalArgumentException("Expected rank 3 input with shape [minibatch, layerSize, tsLength]. Got " +
                    "rank " + in.rank() + " with shape " + Arrays.toString(in.shape()));
        }
        origOutputShape = in.shape();
        boolean nwc = TimeSeriesUtils.getFormatFromRnnLayer(underlying.conf().getLayer()) == RNNFormat.NWC;
//        underlying instanceof  BaseRecurrentLayer && ((BaseRecurrentLayer)underlying).getDataFormat() == RNNFormat.NWC)||
//                underlying instanceof MaskZeroLayer && ((MaskZeroLayer)underlying).getUnderlying() instanceof BaseRecurrentLayer &&
//                        ((BaseRecurrentLayer)((MaskZeroLayer)underlying).getUnderlying()).getDataFormat() == RNNFormat.NWC;
        if (nwc){
            in = in.permute(0, 2, 1);
        }

        INDArray mask = underlying.getMaskArray();
        Pair<INDArray,int[]> p = TimeSeriesUtils.pullLastTimeSteps(in, mask, workspaceMgr, arrayType);
        lastTimeStepIdxs = p.getSecond();

        return p.getFirst();
    }
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);
    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, delta.dataType(), new long[]{w.size(0), delta.size(0)}, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();

    //Normally we would clear weightNoiseParams here - but we want to reuse them for forward + backward + score
    // So this is instead done in MultiLayerNetwork/CompGraph backprop methods

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

@Test
public void testOutputOnly() throws IOException {
    //Nd4jCpu.Environment.getInstance().setUseMKLDNN(false);
    File dir = testDir.newFolder();
    Map<String, INDArray> inputs = TFGraphTestAllHelper.inputVars(modelName, MODEL_DIR, dir);
    Map<String, INDArray> predictions = TFGraphTestAllHelper.outputVars(modelName, MODEL_DIR, dir);
    Pair<Double,Double> precisionOverride = TFGraphTestAllHelper.testPrecisionOverride(modelName);
    Double maxRE = (precisionOverride == null ? null : precisionOverride.getFirst());
    Double minAbs = (precisionOverride == null ? null : precisionOverride.getSecond());

    TFGraphTestAllHelper.checkOnlyOutput(inputs, predictions, modelName, MODEL_DIR, MODEL_FILENAME, executeWith,
            TFGraphTestAllHelper.LOADER, maxRE, minAbs, printArraysDebugging);
}
 

private void preLoadTokens() {
    if (preLoadedTokens != null) {
        return;
    }
    Pair<String, String> p = sentenceProvider.nextSentence();
    List<String> tokens = tokenizeSentence(p.getFirst());
    if (!tokens.isEmpty()) {
        preLoadedTokens = new Pair<>(tokens, p.getSecond());
    }
}
 

@Test
public void testSqueezeExpandChain() {

    val origShape = new long[]{3, 4, 5};

    for (int i = 0; i < 3; i++) {

        val shape = origShape.clone();
        shape[i] = 1;

        for (Pair<INDArray, String> p : NDArrayCreationUtil
                .getAll3dTestArraysWithShape(12345, shape, DataType.FLOAT)) {
            INDArray inArr = p.getFirst().muli(100);

            SameDiff sd = SameDiff.create();
            SDVariable in = sd.var("in", inArr);
            SDVariable squeeze = sd.squeeze(in, i);
            SDVariable expand = sd.expandDims(squeeze, i);

            INDArray out = expand.eval();

            String msg = "expand/Squeeze=" + i + ", source=" + p.getSecond();

            assertEquals(msg, out, inArr);  //squeeze -> expand: should be opposite ops
        }
    }
}
 

@Test
public void testToOffsetZeroCopy() {
    List<Pair<INDArray, String>> testInputs = NDArrayCreationUtil.getAllTestMatricesWithShape(4, 5, 123, DataType.DOUBLE);

    int cnt = 0;
    for (Pair<INDArray, String> pair : testInputs) {
        String msg = pair.getSecond();
        INDArray in = pair.getFirst();
        INDArray dup = Shape.toOffsetZeroCopy(in);
        INDArray dupc = Shape.toOffsetZeroCopy(in, 'c');
        INDArray dupf = Shape.toOffsetZeroCopy(in, 'f');
        INDArray dupany = Shape.toOffsetZeroCopyAnyOrder(in);

        assertEquals(msg + ": " + cnt, in, dup);
        assertEquals(msg, in, dupc);
        assertEquals(msg, in, dupf);
        assertEquals(msg, dupc.ordering(), 'c');
        assertEquals(msg, dupf.ordering(), 'f');
        assertEquals(msg, in, dupany);

        assertEquals(dup.offset(), 0);
        assertEquals(dupc.offset(), 0);
        assertEquals(dupf.offset(), 0);
        assertEquals(dupany.offset(), 0);
        assertEquals(dup.length(), dup.data().length());
        assertEquals(dupc.length(), dupc.data().length());
        assertEquals(dupf.length(), dupf.data().length());
        assertEquals(dupany.length(), dupany.data().length());
        cnt++;
    }
}
 

@Test
public void testCheckpointListenerKeepLast3AndEvery3() throws Exception {
    File f = tempDir.newFolder();
    Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData();
    MultiLayerNetwork net = p.getFirst();
    DataSetIterator iter = p.getSecond();


    CheckpointListener l = new CheckpointListener.Builder(f)
            .keepLastAndEvery(3, 3)
            .saveEveryNEpochs(2)
            .build();
    net.setListeners(l);

    for(int i=0; i<20; i++ ){   //40 iterations total
        net.fit(iter);
    }

    //Expect models saved at end of epochs: 1, 3, 5, 7, 9, 11, 13, 15, 17, 19
    //But: keep only 5, 11, 15, 17, 19
    File[] files = f.listFiles();
    int count = 0;
    Set<Integer> ns = new HashSet<>();
    for(File f2 : files){
        if(!f2.getPath().endsWith(".zip")){
            continue;
        }
        count++;
        int prefixLength = "checkpoint_".length();
        int end = f2.getName().lastIndexOf("_");
        int num = Integer.parseInt(f2.getName().substring(prefixLength, end));

        MultiLayerNetwork n = ModelSerializer.restoreMultiLayerNetwork(f2, true);
        int expEpoch = 2 * (num+1) - 1;
        assertEquals(expEpoch, n.getEpochCount());

        ns.add(n.getEpochCount());
        count++;
    }

    assertEquals(ns.toString(), 5, ns.size());
    assertTrue(ns.toString(), ns.containsAll(Arrays.asList(5, 11, 15, 17, 19)));

    assertEquals(5, l.availableCheckpoints().size());
}
 

public HyperRect(Pair<float[], float[]> ends) {
    this(ends.getFirst(), ends.getSecond());
}
 

public static void checkIntermediate(Map<String, INDArray> inputs, String modelName, String baseDir, String modelFileName,
                                         ExecuteWith execType, BiFunction<File,String,SameDiff> loader,
                                         Double maxRelErrorOverride, Double minAbsErrorOverride, File localTestDir, boolean printArraysDebugging) throws IOException {
        Preconditions.checkArgument((maxRelErrorOverride == null) == (minAbsErrorOverride == null), "Both maxRelErrorOverride and minAbsErrorOverride" +
                " must be null or both must be provided");
        Nd4j.EPS_THRESHOLD = 1e-3;
        OpExecOrderListener listener = new OpExecOrderListener();       //Used to collect exec order
        Pair<SameDiff, Map<String,INDArray>> p = getGraphAfterExec(baseDir, modelFileName, modelName, inputs, execType, loader, Collections.singletonList(listener), null, printArraysDebugging);
        SameDiff graph = p.getFirst();
        Map<String,INDArray> sdPredictions = p.getSecond();

        //Collect coverage info about ops
        OpValidation.collectTensorflowImportCoverage(graph);

        if (!execType.equals(ExecuteWith.JUST_PRINT)) {
            int count = 0;
            //Evaluate the nodes in their execution order - this is useful for debugging (as we want the *first* failure
            // to be detected before later failures)
            List<String> varNames = new ArrayList<>();
            Map<String,SameDiffOp> fns = graph.getOps();
            List<String> execOrder = listener.getOpNamesList();
            for(String opName : execOrder){
                String[] outputs = graph.getOutputsForOp(fns.get(opName).getOp());
                Collections.addAll(varNames, outputs);
            }

            for (String varName : varNames) {
                if (!inputs.containsKey(varName)) { //avoiding placeholders
                    INDArray tfValue = intermediateVars(modelName, baseDir, varName, localTestDir);
                    if (tfValue == null) {
                        continue;
                    }
                    log.info("Starting check: variable {}", varName);
                    if (skipNode(modelName, varName)) {
                        log.info("\n\tFORCING no check on " + varName);
                    } else {
                        assertArrayEquals("Shape not equal on node " + varName, tfValue.shape(), graph.getVariable(varName).getShape());
                        INDArray sdVal = sdPredictions.get(varName);
                        if(maxRelErrorOverride != null){
                            INDArray diff = Transforms.abs(tfValue.sub(sdVal), false);
                            INDArray absErrorMask = diff.gte(minAbsErrorOverride);   //value 1 if x[i] > minAbsError; value 0 otherwise. Used to get rid of 1e-30 vs. 1e-29 type failures
                            INDArray sumAbs = Transforms.abs(tfValue, true).addi(Transforms.abs(sdVal, true));
                            BooleanIndexing.replaceWhere(sumAbs, 1.0, Conditions.equals(0.0));  //Can only get 0.0 if both are zeros - need to avoid 0/0=NaN
                            INDArray relError = diff.divi(sumAbs);
                            relError.muli(absErrorMask);

                            int countExceeds = Nd4j.getExecutioner().exec(new MatchCondition(relError, Conditions.greaterThan(maxRelErrorOverride))).getInt(0);

                            double maxRE = -1;
                            //Mainly used for analysis in debugger:
                            DifferentialFunction op = null;
                            String[] opInputs = null;
                            if(countExceeds > 0){
                                maxRE = relError.maxNumber().doubleValue();
                                //Find the op that this variable is produced by
                                op = graph.getVariableOutputOp(varName);
                                opInputs = graph.getInputsForOp(op);
                            }


                            assertEquals( varName + ": " + countExceeds + " values exceed maxRelError=" + maxRelErrorOverride
                                    + " with minAbsError=" + minAbsErrorOverride + "; largest observed relError=" + maxRE, 0, countExceeds);
                        } else {
//                            assertEquals("Value not equal on node " + varName, tfValue, sdVal);
                            if(tfValue.equals(sdVal)){
                                System.out.println("Pass: " + varName);
                            } else {
                                System.out.println("FAIL: " + varName);
                                System.out.println("TF:\n" + tfValue);
                                System.out.println("SD:\n" + sdVal);
                            }

                        }
                        log.info("Values and shapes equal for {}", varName);
                        count++;
                    }

                }
            }

            assertTrue("No intermediate variables were checked", count > 0);
        }

        Nd4j.EPS_THRESHOLD = 1e-5;
    }
 

private void setShapeInformation(Pair<DataBuffer, long[]> shapeInfo) {
    this.shapeInformation = shapeInfo.getFirst();
    this.jvmShapeInfo = new JvmShapeInfo(shapeInfo.getSecond());
}
 

@Test
public void testBackpropGradient() {
    //Testing: MultiLayerNetwork.backpropGradient()
    //i.e., specifically without an output layer

    int nIn = 10;
    int nOut = 40;
    int miniBatch = 5;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .updater(new Sgd(0.1)).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(2, new DenseLayer.Builder().nIn(30).nOut(nOut).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Nd4j.getRandom().setSeed(12345);
    INDArray eps = Nd4j.rand(miniBatch, nOut);
    INDArray input = Nd4j.rand(miniBatch, nIn);

    net.setInput(input);
    net.feedForward(true, false); //Need to feed forward before backprop

    Pair<Gradient, INDArray> pair = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces());
    INDArray epsOut = pair.getSecond();
    assertNotNull(epsOut);
    assertArrayEquals(new long[] {miniBatch, nIn}, epsOut.shape());

    Gradient g = pair.getFirst();
    Map<String, INDArray> gradMap = g.gradientForVariable();
    assertEquals(6, gradMap.size()); //3 layers, weight + bias gradients for each

    String[] expKeys = {"0_" + DefaultParamInitializer.WEIGHT_KEY, "0_" + DefaultParamInitializer.BIAS_KEY,
                    "1_" + DefaultParamInitializer.WEIGHT_KEY, "2_" + DefaultParamInitializer.BIAS_KEY,
                    "2_" + DefaultParamInitializer.WEIGHT_KEY, "2_" + DefaultParamInitializer.BIAS_KEY};
    Set<String> keys = gradMap.keySet();
    for (String s : expKeys) {
        assertTrue(keys.contains(s));
    }

    /*
    System.out.println(pair);
    
    //Use updater to go from raw gradients -> updates
    //Apply learning rate, gradient clipping, adagrad/momentum/rmsprop etc
    Updater updater = UpdaterCreator.getUpdater(net);
    updater.update(net, g, 0, miniBatch);
    
    StepFunction stepFunction = new NegativeGradientStepFunction();
    INDArray params = net.params();
    System.out.println(Arrays.toString(params.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 10)).dup().data().asFloat()));
    stepFunction.step(params, g.gradient());
    net.setParams(params);    //params() may not be in-place
    System.out.println(Arrays.toString(params.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 10)).dup().data().asFloat()));
    */
}
 

@Test
public void testCheckpointListenerEveryTimeUnit() throws Exception {
    File f = tempDir.newFolder();
    Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData();
    MultiLayerNetwork net = p.getFirst();
    DataSetIterator iter = p.getSecond();


    CheckpointListener l = new CheckpointListener.Builder(f)
            .keepLast(3)
            .saveEvery(4900, TimeUnit.MILLISECONDS)
            .build();
    net.setListeners(l);

    for(int i=0; i<3; i++ ){   //10 iterations total
        net.fit(iter);
        Thread.sleep(5000);
    }

    //Expect models saved at iterations: 2, 4, 6, 8 (iterations 0 and 1 shoud happen before first 3 seconds is up)
    //But: keep only 5, 7, 9
    File[] files = f.listFiles();
    Set<Integer> ns = new HashSet<>();
    for(File f2 : files){
        if(!f2.getPath().endsWith(".zip")){
            continue;
        }

        int prefixLength = "checkpoint_".length();
        int num = Integer.parseInt(f2.getName().substring(prefixLength, prefixLength+1));

        MultiLayerNetwork n = ModelSerializer.restoreMultiLayerNetwork(f2, true);
        int expIter = 2 * (num + 1);
        assertEquals(expIter, n.getIterationCount());

        ns.add(n.getIterationCount());
    }

    assertEquals(2, l.availableCheckpoints().size());
    assertEquals(ns.toString(), 2, ns.size());
    System.out.println(ns);
    assertTrue(ns.containsAll(Arrays.asList(2,4)));
}
 

private static String getTestWithOpsErrorMsg(int i, int j, String op, Pair<INDArray, String> first,
                Pair<INDArray, String> second) {
    return i + "," + j + " - " + first.getSecond() + "." + op + "(" + second.getSecond() + ")";
}
 

/**
 * Constructor from parsed Keras layer configuration dictionary.
 *
 * @param layerConfig           dictionary containing Keras layer configuration
 * @param enforceTrainingConfig whether to enforce training-related configuration options
 * @param previousLayers        - dictionary containing the previous layers in the topology
 * @throws InvalidKerasConfigurationException     Invalid Keras config
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
public KerasLSTM(Map<String, Object> layerConfig, boolean enforceTrainingConfig,
                 Map<String, ? extends KerasLayer> previousLayers)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    super(layerConfig, enforceTrainingConfig);

    IWeightInit init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
            enforceTrainingConfig, conf, kerasMajorVersion);

    IWeightInit recurrentInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(),
            enforceTrainingConfig, conf, kerasMajorVersion);

    boolean hasBias = getHasBiasFromConfig(layerConfig, conf);

    Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
    this.returnSequences = (Boolean) innerConfig.get(conf.getLAYER_FIELD_RETURN_SEQUENCES());

    // TODO: support recurrent dropout
    // double recurrentDropout = KerasRnnUtils.getRecurrentDropout(conf, layerConfig);
    this.unroll = KerasRnnUtils.getUnrollRecurrentLayer(conf, layerConfig);

    LayerConstraint biasConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_B_CONSTRAINT(), conf, kerasMajorVersion);
    LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
    LayerConstraint recurrentConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_RECURRENT_CONSTRAINT(), conf, kerasMajorVersion);

    Pair<Boolean, Double> maskingConfig = KerasLayerUtils.getMaskingConfiguration(inboundLayerNames, previousLayers);

    LSTM.Builder builder = new LSTM.Builder()
            .gateActivationFunction(getGateActivationFromConfig(layerConfig))
            .forgetGateBiasInit(getForgetBiasInitFromConfig(layerConfig, enforceTrainingConfig))
            .name(this.layerName)
            .nOut(getNOutFromConfig(layerConfig, conf))
            .dropOut(this.dropout)
            .activation(getIActivationFromConfig(layerConfig, conf))
            .weightInit(init)
            .weightInitRecurrent(recurrentInit)
            .biasInit(0.0) // TODO: this is incorrect
            .l1(this.weightL1Regularization)
            .l2(this.weightL2Regularization).dataFormat(RNNFormat.NWC);
    Integer nIn = KerasLayerUtils.getNInFromInputDim(layerConfig, conf);
    if(nIn != null)
        builder.setNIn(nIn);
    if (biasConstraint != null)
        builder.constrainBias(biasConstraint);
    if (weightConstraint != null)
        builder.constrainInputWeights(weightConstraint);
    if (recurrentConstraint != null)
        builder.constrainRecurrent(recurrentConstraint);

    this.layer = builder.build();
    if (!returnSequences) {
        this.layer = new LastTimeStep(this.layer);
    }
    if (maskingConfig.getFirst()) {
        this.layer = new MaskZeroLayer(this.layer, maskingConfig.getSecond());
    }
}
 

@Test
public void testSimple() throws Exception {
    //Simple sanity check

    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(250, 200);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
}
 

protected void fillQueue() {
    if (queue.isEmpty()) {
        List<INDArray> ndLabels = null;
        List<INDArray> ndFeatures = null;
        float[][] fLabels = null;
        float[][] fFeatures = null;
        double[][] dLabels = null;
        double[][] dFeatures = null;

        int sampleCount = 0;

        for (int cnt = 0; cnt < batchSize; cnt++) {
            if (iterator.hasNext()) {
                Pair<T, T> pair = iterator.next();
                if (numFeatures < 1) {
                    if (pair.getFirst() instanceof INDArray) {
                        numFeatures = (int) ((INDArray) pair.getFirst()).length();
                        numLabels = (int) ((INDArray) pair.getSecond()).length();
                    } else if (pair.getFirst() instanceof float[]) {
                        numFeatures = ((float[]) pair.getFirst()).length;
                        numLabels = ((float[]) pair.getSecond()).length;
                    } else if (pair.getFirst() instanceof double[]) {
                        numFeatures = ((double[]) pair.getFirst()).length;
                        numLabels = ((double[]) pair.getSecond()).length;
                    }
                }

                if (pair.getFirst() instanceof INDArray) {
                    if (ndLabels == null) {
                        ndLabels = new ArrayList<>();
                        ndFeatures = new ArrayList<>();
                    }
                    ndFeatures.add(((INDArray) pair.getFirst()));
                    ndLabels.add(((INDArray) pair.getSecond()));
                } else if (pair.getFirst() instanceof float[]) {
                    if (fLabels == null) {
                        fLabels = new float[batchSize][];
                        fFeatures = new float[batchSize][];
                    }
                    fFeatures[sampleCount] = (float[]) pair.getFirst();
                    fLabels[sampleCount] = (float[]) pair.getSecond();
                } else if (pair.getFirst() instanceof double[]) {
                    if (dLabels == null) {
                        dLabels = new double[batchSize][];
                        dFeatures = new double[batchSize][];
                    }
                    dFeatures[sampleCount] = (double[]) pair.getFirst();
                    dLabels[sampleCount] = (double[]) pair.getSecond();
                }

                sampleCount += 1;
            } else
                break;
        }

        if (sampleCount == batchSize) {
            INDArray labels = null;
            INDArray features = null;
            if (ndLabels != null) {
                labels = Nd4j.vstack(ndLabels);
                features = Nd4j.vstack(ndFeatures);
            } else if (fLabels != null) {
                labels = Nd4j.create(fLabels);
                features = Nd4j.create(fFeatures);
            } else if (dLabels != null) {
                labels = Nd4j.create(dLabels);
                features = Nd4j.create(dFeatures);
            }

            DataSet dataSet = new DataSet(features, labels);
            try {
                queue.add(dataSet);
            } catch (Exception e) {
                // live with it
            }
        }
    }
}
 

@Test
public void testDeleteExisting() throws Exception {
    File f = tempDir.newFolder();
    Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData();
    MultiLayerNetwork net = p.getFirst();
    DataSetIterator iter = p.getSecond();


    CheckpointListener l = new CheckpointListener.Builder(f)
            .keepAll()
            .saveEveryNEpochs(1)
            .build();
    net.setListeners(l);

    for(int i=0; i<3; i++ ){
        net.fit(iter);
    }

    //Now, create new listener:
    try{
        l = new CheckpointListener.Builder(f)
                .keepAll()
                .saveEveryNEpochs(1)
                .build();
        fail("Expected exception");
    } catch (IllegalStateException e){
        assertTrue(e.getMessage().contains("Use deleteExisting(true)"));
    }

    l = new CheckpointListener.Builder(f)
            .keepAll()
            .saveEveryNEpochs(1)
            .deleteExisting(true)
            .build();
    net.setListeners(l);

    net.fit(iter);

    File[] fList = f.listFiles();   //checkpoint meta file + 1 checkpoint
    assertNotNull(fList);
    assertEquals(2, fList.length);
}
 

@Test
public void testPerformance() throws Exception {

    StopWatch watch = new StopWatch();
    watch.start();
    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(DataType.FLOAT, 250, 20);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
    watch.stop();
    System.out.println("Elapsed time : " + watch);
}
 

@Override
public Long apply(Pair<Long, Long> input) {
    return input.getFirst() + input.getSecond();
}