org.apache.arrow.vector.FieldVector Java Examples

/**
 * Method for reading batches of booleans.
 */
public int nextBatchBoolean(
    final FieldVector vector,
    final int expectedBatchSize,
    final int numValsInVector,
    NullabilityHolder nullabilityHolder) {
  final int actualBatchSize = getActualBatchSize(expectedBatchSize);
  if (actualBatchSize <= 0) {
    return 0;
  }
  vectorizedDefinitionLevelReader
      .readBatchOfBooleans(vector, numValsInVector, actualBatchSize,
          nullabilityHolder, plainValuesReader);
  triplesRead += actualBatchSize;
  this.hasNext = triplesRead < triplesCount;
  return actualBatchSize;
}
 

/**
 * Filter a batch of records against the expression.
 * @param recordCount - number of records to consume
 * @return the number of records that passed the filter
 * @throws GandivaException on evaluation exception.
 */
public int filterBatch(int recordCount) throws GandivaException {
  if (recordCount == 0) {
    return 0;
  }

  root.setRowCount(recordCount);
  List<ArrowBuf> buffers = Lists.newArrayList();
  for (FieldVector v : root.getFieldVectors()) {
    buffers.addAll(v.getFieldBuffers());
  }

  selectionVector.allocateNew(recordCount);

  // do not take ownership of the buffer.
  ArrowBuf svBuffer = selectionVector.getBuffer(false);
  SelectionVector selectionVectorGandiva = new SelectionVectorInt16(svBuffer);

  filter.evaluate(recordCount, buffers, selectionVectorGandiva);
  selectionVector.setRecordCount(selectionVectorGandiva.getRecordCount());
  return selectionVector.getCount();
}
 

private static void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) {
  checkArgument(nodes.hasNext(),
      "no more field nodes for for field " + field + " and vector " + vector);
  ArrowFieldNode fieldNode = nodes.next();
  List<BufferLayout> bufferLayouts = TypeLayout.getTypeLayout(field.getType()).getBufferLayouts();
  List<ArrowBuf> ownBuffers = new ArrayList<>(bufferLayouts.size());
  for (int j = 0; j < bufferLayouts.size(); j++) {
    ownBuffers.add(buffers.next());
  }
  try {
    vector.loadFieldBuffers(fieldNode, ownBuffers);
  } catch (RuntimeException e) {
    throw new IllegalArgumentException("Could not load buffers for field " +
        field + ". error message: " + e.getMessage(), e);
  }
  List<Field> children = field.getChildren();
  if (children.size() > 0) {
    List<FieldVector> childrenFromFields = vector.getChildrenFromFields();
    checkArgument(children.size() == childrenFromFields.size(), "should have as many children as in the schema: found " + childrenFromFields.size() + " expected " + children.size());
    for (int i = 0; i < childrenFromFields.size(); i++) {
      Field child = children.get(i);
      FieldVector fieldVector = childrenFromFields.get(i);
      loadBuffers(fieldVector, child, buffers, nodes);
    }
  }
}
 

private static Block subtract(BlockAllocator allocator, EquatableValueSet left, EquatableValueSet right)
{
    Block resultBlock = BlockUtils.newEmptyBlock(allocator, DEFAULT_COLUMN, left.getType());
    FieldVector result = resultBlock.getFieldVector(DEFAULT_COLUMN);

    Block lhsBlock = left.getValues();

    FieldReader lhs = lhsBlock.getFieldReader(DEFAULT_COLUMN);

    int count = 0;
    for (int i = 0; i < lhsBlock.getRowCount(); i++) {
        lhs.setPosition(i);
        if (!isPresent(lhs.readObject(), right.valueBlock)) {
            BlockUtils.setValue(result, count++, lhs.readObject());
        }
    }
    resultBlock.setRowCount(count);
    return resultBlock;
}
 

private static Block intersect(BlockAllocator allocator, EquatableValueSet left, EquatableValueSet right)
{
    Block resultBlock = BlockUtils.newEmptyBlock(allocator, DEFAULT_COLUMN, left.getType());
    FieldVector result = resultBlock.getFieldVector(DEFAULT_COLUMN);

    Block lhsBlock = left.getValues();

    FieldReader lhs = lhsBlock.getFieldReader(DEFAULT_COLUMN);

    int count = 0;
    for (int i = 0; i < lhsBlock.getRowCount(); i++) {
        lhs.setPosition(i);
        if (isPresent(lhs.readObject(), right.valueBlock)) {
            BlockUtils.setValue(result, count++, lhs.readObject());
        }
    }
    resultBlock.setRowCount(count);
    return resultBlock;
}
 

/**
 * Project the build data (including keys from the probe)
 * @param offsetAddr
 * @param count
 */
private void projectBuild(final long offsetAddr, final int count){
  buildCopyWatch.start();
  if (buildCopiers.size() == 0) {
    // No data in build side
    final List<FieldVector> buildOutputs = this.buildOutputs;
    for (FieldVector fieldVector : buildOutputs) {
      fieldVector.allocateNew();
    }
  } else {
    for (FieldBufferCopier c : buildCopiers) {
      c.copy(offsetAddr, count);
    }
  }
  buildCopyWatch.stop();
}
 

public void load(SerializedField metadata, ArrowBuf buf) {
    final List<SerializedField> fields = metadata.getChildList();
    structVector.valueCount = metadata.getValueCount();

    int bufOffset = 0;
    for (final SerializedField child : fields) {
      final Field fieldDef = SerializedFieldHelper.create(child);

      FieldVector vector = structVector.getChild(fieldDef.getName());
      if (vector == null) {
//         if we arrive here, we didn't have a matching vector.
        vector = BasicTypeHelper.getNewVector(fieldDef, structVector.allocator);
        structVector.putChild(fieldDef.getName(), vector);
      }
      if (child.getValueCount() == 0) {
        vector.clear();
      } else {
        TypeHelper.load(vector, child, buf.slice(bufOffset, child.getBufferLength()));
      }
      bufOffset += child.getBufferLength();
    }

    Preconditions.checkState(bufOffset == buf.capacity());
  }
 

public static void load(RecordBatch recordBatch, VectorAccessible vectorAccessible, ArrowBuf body) {
  List<Field> fields = vectorAccessible.getSchema().getFields();
  List<FieldVector> fieldVectors = FluentIterable.from(vectorAccessible)
    .transform(new Function<VectorWrapper<?>, FieldVector>() {
      @Override
      public FieldVector apply(VectorWrapper<?> wrapper) {
        return (FieldVector) wrapper.getValueVector();
      }
    }).toList();
  try {
    ArrowRecordBatch arrowRecordBatch = deserializeRecordBatch(recordBatch, body);
    Iterator<ArrowFieldNode> nodes = arrowRecordBatch.getNodes().iterator();
    Iterator<ArrowBuf> buffers = arrowRecordBatch.getBuffers().iterator();
    for (int i = 0; i < fields.size(); ++i) {
      Field field = fields.get(i);
      FieldVector fieldVector = fieldVectors.get(i);
      loadBuffers(fieldVector, field, buffers, nodes);
    }
    if (buffers.hasNext()) {
      throw new IllegalArgumentException("not all buffers were consumed. " + buffers);
    }
  } catch (IOException e) {
    throw new RuntimeException("could not deserialize batch for " + vectorAccessible.getSchema(), e);
  }
}
 

@Test
public void testArrowBatchSet() {
    Schema.Builder schema = new Schema.Builder();
    List<String> single = new ArrayList<>();
    for(int i = 0; i < 2; i++) {
        schema.addColumnInteger(String.valueOf(i));
        single.add(String.valueOf(i));
    }

    List<List<Writable>> input = Arrays.asList(
            Arrays.<Writable>asList(new IntWritable(0),new IntWritable(1)),
            Arrays.<Writable>asList(new IntWritable(2),new IntWritable(3))
    );

    List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator,schema.build(),input);
    ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector,schema.build());
    List<Writable> assertion = Arrays.<Writable>asList(new IntWritable(4), new IntWritable(5));
    writableRecordBatch.set(1, Arrays.<Writable>asList(new IntWritable(4),new IntWritable(5)));
    List<Writable> recordTest = writableRecordBatch.get(1);
    assertEquals(assertion,recordTest);
}
 

static ImmutableList<MultiDestCopier> getCopiers(final List<FieldVector> inputs, OutgoingBatch[] batches,
                                                        CopyWatches copyWatches) {
  ImmutableList.Builder<MultiDestCopier> copiers = ImmutableList.builder();
  final int numFields = inputs.size();
  final int numBatches = batches.length;

  // for each field group all corresponding field vectors for all batches together
  // outputs[f][b] = field vector f for batch b
  final FieldVector[][] outputs = new FieldVector[numFields][numBatches];
  for (int b = 0; b < numBatches; b++) {
    final List<FieldVector> fieldVectors = batches[b].getFieldVectors();
    Preconditions.checkArgument(numFields == fieldVectors.size(), "Input and output lists must be same size.");
    for (int f = 0; f < numFields; f++) {
      outputs[f][b] = fieldVectors.get(f);
    }
  }

  for (int i = 0; i < inputs.size(); i++) {
    final FieldVector input = inputs.get(i);
    final FieldVector[] targets = outputs[i];
    addValueCopier(input, i, targets, copiers, copyWatches);
  }
  return copiers.build();
}
 

private void setFixedWidthBinary(
    FieldVector vector, int typeWidth, NullabilityHolder nullabilityHolder,
    int idx, ByteBuffer buffer) {
  vector.getDataBuffer()
      .setBytes(idx * typeWidth, buffer.array(),
          buffer.position() + buffer.arrayOffset(), buffer.limit() - buffer.position());
  setNotNull(vector, nullabilityHolder, idx);
}
 

void readBatchOfDictionaryEncodedFloats(FieldVector vector, int startOffset, int numValuesToRead, Dictionary dict,
                                        NullabilityHolder nullabilityHolder, int typeWidth) {
  int left = numValuesToRead;
  int idx = startOffset;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int num = Math.min(left, this.currentCount);
    switch (mode) {
      case RLE:
        for (int i = 0; i < num; i++) {
          vector.getDataBuffer().setFloat(idx * typeWidth, dict.decodeToFloat(currentValue));
          setNotNull(vector, nullabilityHolder, idx);
          idx++;
        }
        break;
      case PACKED:
        for (int i = 0; i < num; i++) {
          vector.getDataBuffer()
              .setFloat(idx * typeWidth, dict.decodeToFloat(packedValuesBuffer[packedValuesBufferIdx++]));
          setNotNull(vector, nullabilityHolder, idx);
          idx++;
        }
        break;
    }
    left -= num;
    currentCount -= num;
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxMemAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxMemAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    java.math.BigDecimal newVal = DecimalUtils.getBigDecimalFromLEBytes(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new max or existing) at the target location of accumulation vector */
    PlatformDependent.putLong(maxAddr, Double.doubleToLongBits(max(Double.longBitsToDouble(PlatformDependent.getLong(maxAddr)), newVal.doubleValue(), bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

/**
 * An index in to an individual
 * {@link ArrowRecordBatch}
 * @param list the list of field vectors to use
 * @param schema the schema to use
 */
public ArrowWritableRecordTimeSeriesBatch(List<FieldVector> list, Schema schema,int timeSeriesStride) {
    this.list = list;
    this.schema = schema;
    //each column should have same number of rows
    this.timeSeriesStride = timeSeriesStride;
    this.size = list.size() * list.get(0).getValueCount() / timeSeriesStride;

}
 

/**
 * Method for reading a batch of decimals backed by INT32 and INT64 parquet data types. Since Arrow stores all
 * decimals in 16 bytes, byte arrays are appropriately padded before being written to Arrow data buffers.
 */
public int nextBatchIntLongBackedDecimal(
    final FieldVector vector, final int expectedBatchSize, final int numValsInVector,
    final int typeWidth, NullabilityHolder nullabilityHolder) {
  final int actualBatchSize = getActualBatchSize(expectedBatchSize);
  if (actualBatchSize <= 0) {
    return 0;
  }
  if (dictionaryDecodeMode == DictionaryDecodeMode.EAGER) {
    vectorizedDefinitionLevelReader
        .readBatchOfDictionaryEncodedIntLongBackedDecimals(
            vector,
            numValsInVector,
            typeWidth,
            actualBatchSize,
            nullabilityHolder,
            dictionaryEncodedValuesReader,
            dictionary);
  } else {
    vectorizedDefinitionLevelReader.readBatchOfIntLongBackedDecimals(
        vector,
        numValsInVector,
        typeWidth,
        actualBatchSize,
        nullabilityHolder,
        plainValuesReader);
  }
  triplesRead += actualBatchSize;
  this.hasNext = triplesRead < triplesCount;
  return actualBatchSize;
}
 

private TreeNode visitValueVectorReadExpression(ValueVectorReadExpression readExpr, Void value) {

    FieldVector vector = incoming.getValueAccessorById(FieldVector.class,
                                                       readExpr.getTypedFieldId().getFieldIds())
                                 .getValueVector();
    referencedFields.add(vector.getField());
    return TreeBuilder.makeField(vector.getField());
  }
 

public boolean setUpTwister2ArrowWrite(int workerId) throws Exception {
  LOG.fine("%%%%%%%%% worker id details:" + workerId + "\t" + arrowFile);
  this.root = VectorSchemaRoot.create(Schema.fromJSON(arrowSchema), this.rootAllocator);
  Path path = new Path(arrowFile);
  this.fileSystem = FileSystemUtils.get(path);
  this.fsDataOutputStream = fileSystem.create(path);
  this.twister2ArrowOutputStream = new Twister2ArrowOutputStream(this.fsDataOutputStream);
  DictionaryProvider.MapDictionaryProvider provider
      = new DictionaryProvider.MapDictionaryProvider();
  if (!flag) {
    this.arrowFileWriter = new ArrowFileWriter(root, provider,
        this.fsDataOutputStream.getChannel());
  } else {
    this.arrowFileWriter = new ArrowFileWriter(root, provider, this.twister2ArrowOutputStream);
  }

  LOG.info("root schema fields:" + root.getSchema().getFields());
  for (Field field : root.getSchema().getFields()) {
    FieldVector vector = root.getVector(field.getName());
    if (vector.getMinorType().equals(Types.MinorType.INT)) {
      this.generatorMap.put(vector, new IntVectorGenerator());
    } else if (vector.getMinorType().equals(Types.MinorType.BIGINT)) {
      this.generatorMap.put(vector, new BigIntVectorGenerator());
    } else if (vector.getMinorType().equals(Types.MinorType.FLOAT4)) {
      this.generatorMap.put(vector, new FloatVectorGenerator());
    } else {
      throw new RuntimeException("unsupported arrow write type");
    }
  }
  return true;
}
 

public void execute(int recordCount, List<ValueVector> outVectors) throws Exception {
  root.setRowCount(recordCount);

  List<ArrowBuf> buffers = Lists.newArrayList();
  for (FieldVector v : root.getFieldVectors()) {
    buffers.addAll(v.getFieldBuffers());
  }

  projector.evaluate(recordCount, buffers, outVectors);
}
 

public void readBatchOfIntLongBackedDecimals(
    final FieldVector vector, final int startOffset,
    final int typeWidth, final int numValsToRead, NullabilityHolder nullabilityHolder,
    ValuesAsBytesReader valuesReader) {
  int bufferIdx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int num = Math.min(left, this.currentCount);
    byte[] byteArray = new byte[DecimalVector.TYPE_WIDTH];
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          for (int i = 0; i < num; i++) {
            setIntLongBackedDecimal(vector, typeWidth, nullabilityHolder, valuesReader, bufferIdx, byteArray);
            bufferIdx++;
          }
        } else {
          setNulls(nullabilityHolder, bufferIdx, num, vector.getValidityBuffer());
          bufferIdx += num;
        }
        break;
      case PACKED:
        for (int i = 0; i < num; ++i) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            setIntLongBackedDecimal(vector, typeWidth, nullabilityHolder, valuesReader, bufferIdx, byteArray);
          } else {
            setNull(nullabilityHolder, bufferIdx, vector.getValidityBuffer());
          }
          bufferIdx++;
        }
        break;
    }
    left -= num;
    currentCount -= num;
  }
}
 

@Test
public void testReadSchemaAndRecordsFromByteArray() throws Exception {
    BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);

    int valueCount = 3;
    List<Field> fields = new ArrayList<>();
    fields.add(ArrowConverter.field("field1",new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)));
    fields.add(ArrowConverter.intField("field2"));

    List<FieldVector> fieldVectors = new ArrayList<>();
    fieldVectors.add(ArrowConverter.vectorFor(allocator,"field1",new float[] {1,2,3}));
    fieldVectors.add(ArrowConverter.vectorFor(allocator,"field2",new int[] {1,2,3}));


    org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(fields);

    VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, valueCount);
    VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1);
    vectorUnloader.getRecordBatch();
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    try(ArrowFileWriter arrowFileWriter = new ArrowFileWriter(schemaRoot1,null,newChannel(byteArrayOutputStream))) {
        arrowFileWriter.writeBatch();
    } catch (IOException e) {
        log.error("",e);
    }

    byte[] arr = byteArrayOutputStream.toByteArray();
    val arr2 = ArrowConverter.readFromBytes(arr);
    assertEquals(2,arr2.getFirst().numColumns());
    assertEquals(3,arr2.getRight().size());

    val arrowCols = ArrowConverter.toArrowColumns(allocator,arr2.getFirst(),arr2.getRight());
    assertEquals(2,arrowCols.size());
    assertEquals(valueCount,arrowCols.get(0).getValueCount());
}
 

/**
 * Creates an {@link ArrowReader} for the specified {@link VectorSchemaRoot}.
 */
public static RowArrowReader createRowArrowReader(VectorSchemaRoot root, RowType rowType) {
	List<ArrowFieldReader> fieldReaders = new ArrayList<>();
	List<FieldVector> fieldVectors = root.getFieldVectors();
	for (int i = 0; i < fieldVectors.size(); i++) {
		fieldReaders.add(createRowArrowFieldReader(fieldVectors.get(i), rowType.getTypeAt(i)));
	}

	return new RowArrowReader(fieldReaders.toArray(new ArrowFieldReader[0]));
}
 

public static Spread toSpread( final int rowCount , final List<FieldVector> vectorList ){
  Spread spread = new Spread();
  for( ValueVector vector : vectorList ){
    spread.addColumn( ArrowColumnFactory.convert( vector.getField().getName() , vector ) );
  }
  spread.setRowCount( rowCount );
  return spread;
}
 

public BitCopier(FieldVector[] source, FieldVector target, int bufferOrdinal, boolean allocateAsFixed){
  this.source = source;
  this.target = target;
  this.targetAlt = allocateAsFixed ? (FixedWidthVector) target : null;
  this.allocateAsFixed = allocateAsFixed;
  this.bufferOrdinal = bufferOrdinal;
  this.srcAddrs = addresses(bufferOrdinal, source);

}
 

@Override
public Void visitUnknown(LogicalExpression e, StringBuilder sb) {
  if (e instanceof ValueVectorReadExpression) {
    ValueVectorReadExpression readExpression = (ValueVectorReadExpression)e;

    VectorWrapper wrapper = this.incoming.getValueAccessorById(FieldVector.class, readExpression.getFieldId().getFieldIds());
    sb.append(wrapper.getField().getName());
  }

  return null;
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxMemAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxMemAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final double newVal = Double.longBitsToDouble(PlatformDependent.getLong(incomingValue + (incomingIndex * WIDTH_INPUT)));
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    PlatformDependent.putLong(maxAddr, Double.doubleToRawLongBits(max(Double.longBitsToDouble(PlatformDependent.getLong(maxAddr)), newVal, bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

private int load(ArrowBuf buf, int bufOffset, final SerializedField child, FieldVector vector) {
  if (child.getValueCount() == 0) {
    vector.clear();
  } else {
    TypeHelper.load(vector, child, buf.slice(bufOffset, child.getBufferLength()));
  }
  bufOffset += child.getBufferLength();
  return bufOffset;
}
 

/**
 * Used during post-spill processing to convert a float
 * sumzero accumulator to a post-spill double sumzero accumulator
 *
 * @param floatSumZeroAccumulator pre-spill float sumzero accumulator
 * @param input input vector with values to be accumulated
 * @param maxValuesPerBatch max values in a hash table batch
 * @param computationVectorAllocator allocator used for allocating
 *                                   accumulators that store computed values
 */
DoubleSumZeroAccumulator(final SumZeroAccumulators.FloatSumZeroAccumulator floatSumZeroAccumulator,
                         final FieldVector input, final int maxValuesPerBatch,
                         final BufferAllocator computationVectorAllocator) {
  this(input, floatSumZeroAccumulator.getOutput(),
       floatSumZeroAccumulator.getTransferVector(),
       maxValuesPerBatch, computationVectorAllocator,
       floatSumZeroAccumulator.getBitAddresses(),
       floatSumZeroAccumulator.getValueAddresses(),
       floatSumZeroAccumulator.getAccumulators()
  );
}
 

public VariableCopier(FieldVector[] source, FieldVector target) {
  this.source = source;
  this.targetAlt = (VariableWidthVector) target;
  this.target = target;
  this.realloc = Reallocators.getReallocator(target);
  this.srcOffsetAddrs =  addresses(OFFSET_BUFFER_ORDINAL, source);
  this.srcDataAddrs = addresses(2, source);
}
 

@Override
public void consumeDataLeft(int records) throws Exception {
  state.is(State.CAN_CONSUME_L);

  // ensure that none of the variable length vectors are corrupt so we can avoid doing bounds checking later.
  for(FieldVector v : probeVectorsToValidate){
    VariableLengthValidator.validateVariable(v, records);
  }

  state = State.CAN_PRODUCE;
}
 

public void nextBatchTimestampMillis(FieldVector fieldVector, int typeWidth, NullabilityHolder holder) {
  int rowsReadSoFar = 0;
  while (rowsReadSoFar < batchSize && hasNext()) {
    advance();
    int rowsInThisBatch = vectorizedPageIterator.nextBatchTimestampMillis(fieldVector, batchSize - rowsReadSoFar,
        rowsReadSoFar, typeWidth, holder);
    rowsReadSoFar += rowsInThisBatch;
    this.triplesRead += rowsInThisBatch;
    fieldVector.setValueCount(rowsReadSoFar);
  }
}