Java Code Examples for org.apache.arrow.vector.ValueVector#setInitialCapacity()

The following examples show how to use org.apache.arrow.vector.ValueVector#setInitialCapacity() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CompliantTextRecordReader.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  int estimatedRecordCount;
  if ((reader != null) && (reader.getInput() != null) && (vectorMap.size() > 0)) {
    final OptionManager options = context.getOptions();
    final int listSizeEstimate = (int) options.getOption(ExecConstants.BATCH_LIST_SIZE_ESTIMATE);
    final int varFieldSizeEstimate = (int) options.getOption(ExecConstants.BATCH_VARIABLE_FIELD_SIZE_ESTIMATE);
    final int estimatedRecordSize = BatchSchema.estimateRecordSize(vectorMap, listSizeEstimate, varFieldSizeEstimate);
    if (estimatedRecordSize > 0) {
      estimatedRecordCount = (int) Math.min(reader.getInput().length / estimatedRecordSize, numRowsPerBatch);
    } else {
      estimatedRecordCount = (int) numRowsPerBatch;
    }
  } else {
    estimatedRecordCount = (int) numRowsPerBatch;
  }
  for (final ValueVector v : vectorMap.values()) {
    v.setInitialCapacity(estimatedRecordCount);
    v.allocateNew();
  }
}
Example 2
Source File: PartitionerTemplate.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
/**
 * Initialize the OutgoingBatch based on the current schema in incoming RecordBatch
 */
public void initializeBatch() {
  for (VectorWrapper<?> v : incoming) {
    // create new vector
    ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator);
    outgoingVector.setInitialCapacity(maxRecordCount);
    vectorContainer.add(outgoingVector);
  }
  vectorContainer.allocateNew();
  doSetup(incoming, vectorContainer);
}
Example 3
Source File: OutgoingBatch.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
OutgoingBatch(int batchIdx, int nextBatchIdx, int maxRecords, final VectorAccessible incoming,
              BufferAllocator allocator, AccountingExecTunnel tunnel, HashPartitionSender config,
              OperatorContext context, int oppositeMinorFragmentId, OperatorStats stats) {
  Preconditions.checkArgument(maxRecords <= Character.MAX_VALUE, "maxRecords cannot exceed " + Character.MAX_VALUE);
  this.batchIdx = batchIdx;
  this.nextBatchIdx = nextBatchIdx;
  this.maxRecords = maxRecords;

  this.tunnel = tunnel;
  this.config = config;
  this.context = context;
  this.oppositeMinorFragmentId = oppositeMinorFragmentId;

  this.stats = stats;

  for (VectorWrapper<?> v : incoming) {
    ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator);
    outgoingVector.setInitialCapacity(maxRecords);
    add(outgoingVector);

    if (outgoingVector instanceof VarBinaryVector) {
      varbins.add(((VarBinaryVector) outgoingVector));
    } else if (outgoingVector instanceof VarCharVector) {
      varchars.add(((VarCharVector) outgoingVector));
    }
  }
}
Example 4
Source File: PriorityQueueCopierTemplate.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private void allocateVectors(int targetRecordCount) {
  boolean memoryAllocated = false;
  double density = lastSuccessfulDensity;
  while (!memoryAllocated) {
    try {
      for (VectorWrapper<?> w : outgoing) {
        final ValueVector v = w.getValueVector();
        if (v instanceof DensityAwareVector) {
          ((DensityAwareVector) v).setInitialCapacity(targetRecordCount, density);
        } else {
          v.setInitialCapacity(targetRecordCount);
        }
        v.allocateNew();
      }
      memoryAllocated = true;
      lastSuccessfulDensity = density;
    } catch (OutOfMemoryException ex) {
      // halve the density and try again
      density = density / 2;
      if (density < 0.01) {
        logger.debug("PriorityQueueCopierTemplate ran out of memory to allocate outgoing batch. " +
            "Records: {}, density: {}", targetRecordCount, density);
        throw ex;
      }
      // try allocating again with lower density
      logger.debug("PriorityQueueCopierTemplate: Ran out of memory. Retrying allocation with lower density.");
    }
  }
}
Example 5
Source File: VectorContainer.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public void setInitialCapacity(int initialCapacity) {
  for (VectorWrapper<?> w : wrappers) {
    final ValueVector vv = w.getValueVector();
    vv.setInitialCapacity(initialCapacity);
  }
}