Java Code Examples for org.nd4j.linalg.dataset.api.iterator.DataSetIterator#resetSupported()

public AsyncDataSetIterator(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue,
                            boolean useWorkspace, DataSetCallback callback, Integer deviceId) {
    if (queueSize < 2)
        queueSize = 2;

    this.deviceId = deviceId;
    this.callback = callback;
    this.useWorkspace = useWorkspace;
    this.buffer = queue;
    this.prefetchSize = queueSize;
    this.backedIterator = iterator;
    this.workspaceId = "ADSI_ITER-" + java.util.UUID.randomUUID().toString();

    if (iterator.resetSupported() && !iterator.hasNext())
        this.backedIterator.reset();

    this.thread = new AsyncPrefetchThread(buffer, iterator, terminator, null, deviceId);

    thread.setDaemon(true);
    thread.start();
}
 

/**
 * The only constructor
 *
 * @param baseIterator - iterator to be wrapped and split
 * @param totalBatches - total batches in baseIterator
 * @param ratio - train/test split ratio
 */
public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, long totalBatches, double ratio) {
    if (!(ratio > 0.0 && ratio < 1.0))
        throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");

    if (totalBatches < 0)
        throw new ND4JIllegalStateException("totalExamples number should be positive value");

    if (!baseIterator.resetSupported())
        throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");


    this.backedIterator = baseIterator;
    this.totalExamples = totalBatches;
    this.ratio = ratio;
    this.ratios = null;
    this.numTrain = (long) (totalExamples * ratio);
    this.numTest = totalExamples - numTrain;
    this.numArbitrarySets = 2;
    this.splits = null;

    log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
 

/**
 * Perform layerwise unsupervised training on a single pre-trainable layer in the network (VAEs, Autoencoders, etc)
 * for the specified number of epochs<br>
 * If the specified layer index (0 to numLayers - 1) is not a pretrainable layer, this is a no-op.
 *
 * @param layerIdx  Index of the layer to train (0 to numLayers-1)
 * @param iter      Training data
 * @param numEpochs Number of epochs to fit the specified layer for
 */
public void pretrainLayer(int layerIdx, DataSetIterator iter, int numEpochs) {
    Preconditions.checkState(numEpochs > 0, "Number of epochs (%s) must be a positive number", numEpochs);

    if (flattenedGradients == null) {
        initGradientsView();
    }
    if (layerIdx >= layers.length) {
        throw new IllegalArgumentException(
                "Cannot pretrain layer: layerIdx (" + layerIdx + ") >= numLayers (" + layers.length + ")");
    }

    Layer layer = layers[layerIdx];
    if (!layer.isPretrainLayer())
        return;

    if(numEpochs > 1 && !iter.resetSupported())
        throw new IllegalStateException("Cannot fit multiple epochs (" + numEpochs + ") on an iterator that doesn't support resetting");

    if (!iter.hasNext() && iter.resetSupported()) {
        iter.reset();
    }

    log.info("Starting unsupervised training on layer " + layerIdx + " for " + numEpochs + " epochs");
    for(int i=0; i<numEpochs; i++ ) {
        if(i > 0)
            iter.reset();

        while (iter.hasNext()) {
            DataSet next = iter.next();
            input = next.getFeatures();
            pretrainLayer(layerIdx, input);
        }
    }

    int ec = getLayer(layerIdx).conf().getEpochCount() + 1;
    getLayer(layerIdx).conf().setEpochCount(ec);
}
 

public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, long totalBatches, double[] ratios) {
    for (double ratio : ratios) {
        if (!(ratio > 0.0 && ratio < 1.0))
            throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");
    }

    if (totalBatches < 0)
        throw new ND4JIllegalStateException("totalExamples number should be positive value");

    if (!baseIterator.resetSupported())
        throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");


    this.backedIterator = baseIterator;
    this.totalExamples = totalBatches;
    this.ratio = 0.0;
    this.ratios = ratios;
    this.numTrain = 0; //(long) (totalExamples * ratio);
    this.numTest = 0; //totalExamples - numTrain;
    this.numArbitrarySets = ratios.length;

    this.splits = new int[this.ratios.length];
    for (int i = 0; i < this.splits.length; ++i) {
        this.splits[i] = (int)(totalExamples * ratios[i]);
    }

    log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
}
 

public DataSetIteratorSplitter(@NonNull DataSetIterator baseIterator, int[] splits) {

        /*if (!(simpleRatio > 0.0 && simpleRatio < 1.0))
           throw new ND4JIllegalStateException("Ratio value should be in range of 0.0 > X < 1.0");*/

        int totalBatches = 0;
        for (val v:splits)
            totalBatches += v;

        if (totalBatches < 0)
            throw new ND4JIllegalStateException("totalExamples number should be positive value");

        if (!baseIterator.resetSupported())
            throw new ND4JIllegalStateException("Underlying iterator doesn't support reset, so it can't be used for runtime-split");


        this.backedIterator = baseIterator;
        this.totalExamples = totalBatches;
        this.ratio = 0.0;
        this.ratios = null;

        this.numTrain = 0; //(long) (totalExamples * ratio);
        this.numTest = 0; //totalExamples - numTrain;
        this.splits = splits;
        this.numArbitrarySets = splits.length;

        log.warn("IteratorSplitter is used: please ensure you don't use randomization/shuffle in underlying iterator!");
    }
 

public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace,
                DataSetCallback callback, Integer deviceId) {
    this();

    if (queueSize < 2)
        queueSize = 2;

    this.deviceId = deviceId;
    this.callback = callback;
    this.useWorkspace = useWorkspace;
    this.buffer = queue;
    this.prefetchSize = queueSize;
    this.backedIterator = iterator;
    this.workspaceId = "SADSI_ITER-" + java.util.UUID.randomUUID().toString();

    if (iterator.resetSupported())
        this.backedIterator.reset();

    context = TaskContext.get();

    this.thread = new SparkPrefetchThread(buffer, iterator, terminator, null, Nd4j.getAffinityManager().getDeviceForCurrentThread());

    /**
     * We want to ensure, that background thread will have the same thread->device affinity, as master thread
     */

    thread.setDaemon(true);
    thread.start();
}
 

/**
 * The method to use when making predictions for test instances.
 *
 * @param insts the instances to get predictions for
 * @return the class probability estimates (if the class is nominal) or the numeric predictions
 * (if it is numeric)
 * @throws Exception if something goes wrong at prediction time
 */
@Override
public double[][] distributionsForInstances(Instances insts) throws Exception {

  log.info("Calc. dist for {} instances", insts.numInstances());

  // Do we only have a ZeroR model?
  if (zeroR != null) {
    return zeroR.distributionsForInstances(insts);
  }

  // Process input data to have the same filters applied as the training data
  insts = applyFilters(insts);

  // Get predictions
  final DataSetIterator it = getDataSetIterator(insts, CacheMode.NONE);
  double[][] preds = new double[insts.numInstances()][insts.numClasses()];

  if (it.resetSupported()) {
    it.reset();
  }

  int offset = 0;
  boolean next = it.hasNext();

  // Get predictions batch-wise
  while (next) {
    final DataSet ds = Utils.getNext(it);
    final INDArray features = ds.getFeatures();
    final INDArray labelsMask = ds.getLabelsMaskArray();
    INDArray lastTimeStepIndices;
    if (labelsMask != null) {
      lastTimeStepIndices = Nd4j.argMax(labelsMask, 1);
    } else {
      lastTimeStepIndices = Nd4j.zeros(features.size(0), 1);
    }
    INDArray predBatch = model.outputSingle(features);
    int currentBatchSize = (int) predBatch.size(0);
    for (int i = 0; i < currentBatchSize; i++) {
      int thisTimeSeriesLastIndex = lastTimeStepIndices.getInt(i);
      INDArray thisExampleProbabilities =
          predBatch.get(
              NDArrayIndex.point(i),
              NDArrayIndex.all(),
              NDArrayIndex.point(thisTimeSeriesLastIndex));
      for (int j = 0; j < insts.numClasses(); j++) {
        preds[i + offset][j] = thisExampleProbabilities.getDouble(j);
      }
    }

    offset += currentBatchSize; // add batchsize as offset
    boolean iteratorHasInstancesLeft = offset < insts.numInstances();
    next = it.hasNext() || iteratorHasInstancesLeft;
  }

  // Fix classes
  for (int i = 0; i < preds.length; i++) {
    if (preds[i].length > 1) {
      weka.core.Utils.normalize(preds[i]);
    } else {
      // Rescale numeric classes with the computed coefficients in the initialization phase
      preds[i][0] = preds[i][0] * x1 + x0;
    }
  }
  return preds;
}
 

/**
 * The method to use when making predictions for test instances.
 *
 * @param insts the instances to get predictions for
 * @return the class probability estimates (if the class is nominal) or the numeric predictions
 * (if it is numeric)
 * @throws Exception if something goes wrong at prediction time
 */
@Override
public double[][] distributionsForInstances(Instances insts) throws Exception {

  log.info("Calc. dist for {} instances", insts.numInstances());

  // Do we only have a ZeroR model?
  if (zeroR != null) {
    return zeroR.distributionsForInstances(insts);
  }

  // Process input data to have the same filters applied as the training data
  insts = applyFilters(insts);

  // Get predictions
  final DataSetIterator it = getDataSetIterator(insts, CacheMode.NONE);
  double[][] preds = new double[insts.numInstances()][insts.numClasses()];

  if (it.resetSupported()) {
    it.reset();
  }

  int offset = 0;
  boolean next = it.hasNext();

  // Get predictions batch-wise
  while (next) {
    final DataSet ds = Utils.getNext(it);
    final INDArray features = ds.getFeatures();
    final INDArray labelsMask = ds.getLabelsMaskArray();
    INDArray lastTimeStepIndices;
    if (labelsMask != null) {
      lastTimeStepIndices = Nd4j.argMax(labelsMask, 1);
    } else {
      lastTimeStepIndices = Nd4j.zeros(features.size(0), 1);
    }
    INDArray predBatch = model.outputSingle(features);
    int currentBatchSize = (int) predBatch.size(0);
    for (int i = 0; i < currentBatchSize; i++) {
      int thisTimeSeriesLastIndex = lastTimeStepIndices.getInt(i);
      INDArray thisExampleProbabilities =
          predBatch.get(
              NDArrayIndex.point(i),
              NDArrayIndex.all(),
              NDArrayIndex.point(thisTimeSeriesLastIndex));
      for (int j = 0; j < insts.numClasses(); j++) {
        preds[i + offset][j] = thisExampleProbabilities.getDouble(j);
      }
    }

    offset += currentBatchSize; // add batchsize as offset
    boolean iteratorHasInstancesLeft = offset < insts.numInstances();
    next = it.hasNext() || iteratorHasInstancesLeft;
  }

  // Fix classes
  for (int i = 0; i < preds.length; i++) {
    if (preds[i].length > 1) {
      weka.core.Utils.normalize(preds[i]);
    } else {
      // Rescale numeric classes with the computed coefficients in the initialization phase
      preds[i][0] = preds[i][0] * x1 + x0;
    }
  }
  return preds;
}