Java Code Examples for org.apache.arrow.vector.FieldVector#getValidityBufferAddress()

public void accumulate(final long memoryAddr, final int count) {
  final long maxAddr = memoryAddr + count * WIDTH_ORDINAL;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();

  int incomingIndex = 0;
  for(long ordinalAddr = memoryAddr; ordinalAddr < maxAddr; ordinalAddr += WIDTH_ORDINAL, incomingIndex++) {
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    java.math.BigDecimal newVal = DecimalAccumulatorUtilsNoSpill.getBigDecimal(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    final int tableIndex = PlatformDependent.getInt(ordinalAddr);
    final long sumAddr = valueAddresses[tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK] + (tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK) * WIDTH_ACCUMULATOR;
    PlatformDependent.putLong(sumAddr, Double.doubleToLongBits(Double.longBitsToDouble(PlatformDependent.getLong(sumAddr)) + newVal.doubleValue() * bitVal));
  }
}
 

public void accumulate(final long memoryAddr, final int count) {
  final long maxAddr = memoryAddr + count * WIDTH_ORDINAL;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();

  int incomingIndex = 0;
  for(long ordinalAddr = memoryAddr; ordinalAddr < maxAddr; ordinalAddr += WIDTH_ORDINAL, incomingIndex++) {
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    // without if we would need to do a multiply which is slow.
    if (bitVal == 0) {
      continue;
    }
    java.math.BigDecimal newVal = DecimalAccumulatorUtilsNoSpill.getBigDecimal(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    final int tableIndex = PlatformDependent.getInt(ordinalAddr);
    final long sumAddr = valueAddresses[tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK] + (tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK) * WIDTH_ACCUMULATOR;
    BigDecimal curVal = DecimalUtils.getBigDecimalFromLEBytes(sumAddr, valBuf, scale);
    BigDecimal runningVal = curVal.add(newVal, DecimalUtils.MATH);
    byte [] valueInArrowBytes = DecimalUtils.convertBigDecimalToArrowByteArray(runningVal);
    PlatformDependent.copyMemory(valueInArrowBytes, 0, sumAddr, WIDTH_INPUT);
  }
}
 

private void addBatchHelper(final ArrowBuf dataBuffer, final ArrowBuf validityBuffer) {
  resizeAttempted = true;
  /* the memory for accumulator vector comes from the operator's allocator that manages
   * memory for all data structures required for hash agg processing.
   * in other words, we should not use output.getAllocator() here as that allocator is
   * _only_ for for managing memory of output batch.
   */
  FieldVector vector = (FieldVector) output.getTransferPair(computationVectorAllocator).getTo();
  /* store the new vector and increment batches before allocating memory */
  accumulators[batches] = vector;
  final int oldBatches = batches;
  batches++;
  /* if this step or memory allocation inside any child of NestedAccumulator fails,
   * we have captured enough info to rollback the operation.
   */
  loadAccumulatorForNewBatch(vector, dataBuffer, validityBuffer);
  /* need to clear the data since allocate new doesn't do so and we want to start with clean memory */
  initialize(vector);
  bitAddresses[oldBatches] = vector.getValidityBufferAddress();
  valueAddresses[oldBatches] = vector.getDataBufferAddress();

  checkNotNull();
}
 

public void updateTargets(int index, FieldVector target) {
  switch (srcBufferIdx) {
    case 0:
      dstAddrs[index] = target.getValidityBufferAddress();
      break;
    case 1:
      if (target instanceof VariableWidthVector) {
        dstAddrs[index] = target.getOffsetBufferAddress();
      } else {
        dstAddrs[index] = target.getDataBufferAddress();
      }
      break;
    case 2:
      dstAddrs[index] = target.getDataBufferAddress();
      break;
    default:
      throw new UnsupportedOperationException("unexpected buffer offset");
  }
}
 

public void accumulate(final long memoryAddr, final int count) {
  final long maxAddr = memoryAddr + count * WIDTH_ORDINAL;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();

  int incomingIndex = 0;
  for (long ordinalAddr = memoryAddr; ordinalAddr < maxAddr; ordinalAddr += WIDTH_ORDINAL, incomingIndex++) {
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    java.math.BigDecimal newVal = DecimalAccumulatorUtilsNoSpill.getBigDecimal(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    final int tableIndex = PlatformDependent.getInt(ordinalAddr);
    int chunkIndex = tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK;
    int chunkOffset = tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK;
    final long sumAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    PlatformDependent.putLong(sumAddr, Double.doubleToLongBits(Double.longBitsToDouble(PlatformDependent.getLong(sumAddr)) + newVal.doubleValue() * bitVal));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count) {
  final long maxAddr = memoryAddr + count * WIDTH_ORDINAL;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();

  int incomingIndex = 0;
  for (long ordinalAddr = memoryAddr; ordinalAddr < maxAddr; ordinalAddr += WIDTH_ORDINAL, incomingIndex++) {
    java.math.BigDecimal newVal = DecimalAccumulatorUtilsNoSpill.getBigDecimal(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    final int tableIndex = PlatformDependent.getInt(ordinalAddr);
    int chunkIndex = tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK;
    int chunkOffset = tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK;
    final long minAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    PlatformDependent.putLong(minAddr, Double.doubleToLongBits(min(Double.longBitsToDouble(PlatformDependent.getLong(minAddr)), newVal.doubleValue(), bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count) {
  final long maxMemAddr = memoryAddr + count * WIDTH_ORDINAL;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;

  int incomingIndex = 0;
  for(long ordinalAddr = memoryAddr; ordinalAddr < maxMemAddr; ordinalAddr += WIDTH_ORDINAL, incomingIndex++){
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    if (bitVal == 0) {
      continue;
    }
    /* get the corresponding data from input vector -- source data for accumulation */
    long addressOfInput = incomingValue + (incomingIndex * WIDTH_INPUT);
    long newValLow = PlatformDependent.getLong(addressOfInput);
    long newValHigh = PlatformDependent.getLong(addressOfInput + DecimalUtils.LENGTH_OF_LONG);
    final int tableIndex = PlatformDependent.getInt(ordinalAddr);
    int chunkIndex = tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK;
    int chunkOffset = tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK;
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);

    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new max or existing) at the target location of accumulation vector */
    long curValLow = PlatformDependent.getLong(maxAddr);
    long curValHigh = PlatformDependent.getLong(maxAddr + DecimalUtils.LENGTH_OF_LONG);
    int compare = DecimalUtils.compareDecimalsAsTwoLongs(newValHigh, newValLow, curValHigh, curValLow);

    if (compare > 0) {
      /* store the accumulated values(new max or existing) at the target location of
      accumulation vector */
      PlatformDependent.putLong(maxAddr, newValLow);
      PlatformDependent.putLong(maxAddr + 8, newValHigh);
      PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
    }
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final int newVal = PlatformDependent.getInt(incomingValue + (incomingIndex * WIDTH_INPUT));
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long minAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new min or existing) at the target location of accumulation vector */
    PlatformDependent.putInt(minAddr, min(PlatformDependent.getInt(minAddr), newVal, bitVal));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

@Override
public void accumulate(final long memoryAddr, final int count) {
  final long maxAddr = memoryAddr + count * WIDTH_ORDINAL;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final int scale = ((DecimalVector)inputVector).getScale();

  int incomingIndex = 0;
  for (long ordinalAddr = memoryAddr; ordinalAddr < maxAddr; ordinalAddr += WIDTH_ORDINAL, incomingIndex++) {
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;

    //incoming record is null, skip it
    if (bitVal == 0) {
      continue;
    }

    java.math.BigDecimal newVal = DecimalAccumulatorUtilsNoSpill.getBigDecimal(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);

    //get the proper chunk from the ordinal
    final int tableIndex = PlatformDependent.getInt(ordinalAddr);
    int chunkIndex = tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK;
    int chunkOffset = tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK;

    final HllAccumHolder ah =  this.accumulators[chunkIndex];
    final HllSketch sketch = ah.getAccums()[chunkOffset];
    sketch.update(newVal.doubleValue());
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final float newVal = Float.intBitsToFloat(PlatformDependent.getInt(incomingValue + (incomingIndex * WIDTH_INPUT)));
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long minAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new min or existing) at the target location of accumulation vector */
    PlatformDependent.putInt(minAddr, Float.floatToIntBits(min(Float.intBitsToFloat(PlatformDependent.getInt(minAddr)), newVal, bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector) inputVector).getScale();
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    long addressOfInput = incomingValue + (incomingIndex * WIDTH_INPUT);
    long newValLow = PlatformDependent.getLong(addressOfInput) * bitVal;
    long newValHigh =
      PlatformDependent.getLong(addressOfInput + DecimalUtils.LENGTH_OF_LONG) * bitVal;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long sumAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values at the target location of accumulation vector */
    long curValLow = PlatformDependent.getLong(sumAddr);
    long curValHigh = PlatformDependent.getLong(sumAddr + DecimalUtils.LENGTH_OF_LONG);
    DecimalUtils.addSignedDecimals(sumAddr, newValLow, newValHigh, curValLow, curValHigh);
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    java.math.BigDecimal newVal = DecimalUtils.getBigDecimalFromLEBytes(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long minAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new min or existing) at the target location of accumulation vector */
    PlatformDependent.putLong(minAddr, Double.doubleToLongBits(min(Double.longBitsToDouble(PlatformDependent.getLong(minAddr)), newVal.doubleValue(), bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector) inputVector).getScale();
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    java.math.BigDecimal newVal = DecimalUtils.getBigDecimalFromLEBytes(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long sumAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values at the target location of accumulation vector */
    PlatformDependent.putLong(sumAddr, Double.doubleToLongBits(Double.longBitsToDouble(PlatformDependent.getLong(sumAddr)) + newVal.doubleValue() * bitVal));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] valueAddresses = this.valueAddresses;
  final int scale = ((DecimalVector)inputVector).getScale();
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    java.math.BigDecimal newVal = DecimalUtils.getBigDecimalFromLEBytes(incomingValue + (incomingIndex * WIDTH_INPUT), valBuf, scale);
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target address of accumulation vector */
    final long sumAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    /* store the accumulated values at the target location of accumulation vector */
    PlatformDependent.putLong(sumAddr, Double.doubleToLongBits(Double.longBitsToDouble(PlatformDependent.getLong(sumAddr)) + newVal.doubleValue() * bitVal));
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxMemAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxMemAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final double newVal = Double.longBitsToDouble(PlatformDependent.getLong(incomingValue + (incomingIndex * WIDTH_INPUT)));
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    PlatformDependent.putLong(maxAddr, Double.doubleToRawLongBits(max(Double.longBitsToDouble(PlatformDependent.getLong(maxAddr)), newVal, bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxMemAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxMemAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final long newVal = PlatformDependent.getLong(incomingValue + (incomingIndex * WIDTH_INPUT));
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new max or existing) at the target location of accumulation vector */
    PlatformDependent.putLong(maxAddr, max(PlatformDependent.getLong(maxAddr), newVal, bitVal));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxMemAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxMemAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    /* get the corresponding data from input vector -- source data for accumulation */
    final float newVal = Float.intBitsToFloat(PlatformDependent.getInt(incomingValue + (incomingIndex * WIDTH_INPUT)));
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new max or existing) at the target location of accumulation vector */
    PlatformDependent.putInt(maxAddr, Float.floatToRawIntBits(max(Float.intBitsToFloat(PlatformDependent.getInt(maxAddr)), newVal, bitVal)));
    PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
  }
}
 

public void accumulate(final long memoryAddr, final int count,
                       final int bitsInChunk, final int chunkOffsetMask) {
  final long maxMemAddr = memoryAddr + count * PARTITIONINDEX_HTORDINAL_WIDTH;
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long[] bitAddresses = this.bitAddresses;
  final long[] valueAddresses = this.valueAddresses;
  final int maxValuesPerBatch = super.maxValuesPerBatch;

  for (long partitionAndOrdinalAddr = memoryAddr; partitionAndOrdinalAddr < maxMemAddr; partitionAndOrdinalAddr += PARTITIONINDEX_HTORDINAL_WIDTH) {
    /* get the hash table ordinal */
    final int tableIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + HTORDINAL_OFFSET);
    /* get the index of data in input vector */
    final int incomingIndex = PlatformDependent.getInt(partitionAndOrdinalAddr + KEYINDEX_OFFSET);
    final int bitVal = (PlatformDependent.getByte(incomingBit + ((incomingIndex >>> 3))) >>> (incomingIndex & 7)) & 1;
    // no point continuing.
    if (bitVal == 0) {
      continue;
    }
    /* get the corresponding data from input vector -- source data for accumulation */
    long addressOfInput = incomingValue + (incomingIndex * WIDTH_INPUT);
    long newValLow = PlatformDependent.getLong(addressOfInput);
    long newValHigh = PlatformDependent.getLong(addressOfInput + DecimalUtils.LENGTH_OF_LONG);
    /* get the hash table batch index */
    final int chunkIndex = tableIndex >>> bitsInChunk;
    final int chunkOffset = tableIndex & chunkOffsetMask;
    /* get the target addresses of accumulation vector */
    final long maxAddr = valueAddresses[chunkIndex] + (chunkOffset) * WIDTH_ACCUMULATOR;
    final long bitUpdateAddr = bitAddresses[chunkIndex] + ((chunkOffset >>> 5) * 4);
    final int bitUpdateVal = bitVal << (chunkOffset & 31);
    /* store the accumulated values(new max or existing) at the target location of accumulation vector */
    long curValLow = PlatformDependent.getLong(maxAddr);
    long curValHigh = PlatformDependent.getLong(maxAddr + DecimalUtils.LENGTH_OF_LONG);
    int compare = DecimalUtils.compareDecimalsAsTwoLongs(newValHigh, newValLow, curValHigh,
      curValLow);

    if (compare > 0) {
      /* store the accumulated values(new max or existing) at the target location of
      accumulation vector */
      PlatformDependent.putLong(maxAddr, newValLow);
      PlatformDependent.putLong(maxAddr + DecimalUtils.LENGTH_OF_LONG, newValHigh);
      PlatformDependent.putInt(bitUpdateAddr, PlatformDependent.getInt(bitUpdateAddr) | bitUpdateVal);
    }
  }
}
 

@Override
public void accumulate(final long memoryAddr, final int count) {
  FieldVector inputVector = getInput();
  final long incomingBit = inputVector.getValidityBufferAddress();
  final long incomingValue = inputVector.getDataBufferAddress();
  final long numWords = (count + (BITS_PER_LONG-1)) >>> BITS_PER_LONG_SHIFT; // rounded up number of words that cover 'count' bits
  final long maxInputAddr = incomingValue + numWords * WIDTH_LONG;
  final long maxOrdinalAddr = memoryAddr + count * WIDTH_ORDINAL;

  // Like every accumulator, the code below essentially implements:
  //   accumulators[ordinals[i]] += inputs[i]
  // with the only complication that both accumulators and inputs are bits.
  // There's nothing we can do about the locality of the accumulators, but inputs can be processed a word at a time.
  // Algorithm:
  // - get 64 bits worth of inputs, until all inputs exhausted. For each long:
  //   - find the accumulator word it pertains to
  //   - read/update/write the accumulator bit
  // In the code below:
  // - input* refers to the data values in the incoming batch
  // - ordinal* refers to the temporary table that hashAgg passes in, identifying which hash table entry each input matched to
  for (long inputAddr = incomingValue, inputBitAddr = incomingBit, batchCount = 0;
       inputAddr < maxInputAddr;
       inputAddr += WIDTH_LONG, inputBitAddr += WIDTH_LONG, batchCount++) {
    final long inputBatch = PlatformDependent.getLong(inputAddr);
    final long inputBits = PlatformDependent.getLong(inputBitAddr);
    long ordinalAddr = memoryAddr + (batchCount << BITS_PER_LONG_SHIFT);

    for (long bitNum = 0; bitNum < BITS_PER_LONG && ordinalAddr < maxOrdinalAddr; bitNum++, ordinalAddr += WIDTH_ORDINAL) {
      final int tableIndex = PlatformDependent.getInt(ordinalAddr);
      int chunkIndex = tableIndex >>> LBlockHashTableNoSpill.BITS_IN_CHUNK;
      int chunkOffset = tableIndex & LBlockHashTableNoSpill.CHUNK_OFFSET_MASK;

      final int inputBitVal = (int)((inputBits >>> bitNum) & 0x01);
      final int inputVal = (int)((inputBatch >>> bitNum) & 0x01);
      final int minBitUpdateVal = inputBitVal << (chunkOffset & 31);

      final HllAccumHolder ah =  this.accumulators[chunkIndex];
      final HllSketch sketch = ah.getAccums()[chunkOffset];
      sketch.update(inputBitVal);
    }

  }
}
 

/**
 * Take the accumulator vector (the vector that stores computed values)
 * for a particular batch (identified by batchIndex) and output its contents.
 * Output is done by transferring the contents from accumulator vector
 * to its counterpart in outgoing container. As part of transfer,
 * the memory ownership (along with data) is transferred from source
 * vector's allocator to target vector's allocator and source vector's
 * memory is released.
 *
 * While the transfer is good as it essentially
 * avoids copy, we still want the memory associated with
 * allocator of source vector because of post-spill processing
 * where this accumulator vector will still continue to store
 * the computed values as we start treating spilled batches
 * as new input into the operator.
 *
 * This is why we need to immediately allocate
 * the accumulator vector after transfer is done. However we do this
 * for a singe batch only as once we are done outputting a partition,
 * we anyway get rid of all but 1 batch.
 *
 * @param batchIndex batch to output
 */
@Override
public void output(final int batchIndex) {
  final FieldVector accumulationVector = accumulators[batchIndex];
  final TransferPair transferPair = accumulationVector.makeTransferPair(transferVector);
  transferPair.transfer();
  if (batchIndex == 0) {
    ((FixedWidthVector) accumulationVector).allocateNew(maxValuesPerBatch);
    accumulationVector.setValueCount(0);
    initialize(accumulationVector);
    bitAddresses[batchIndex] = accumulationVector.getValidityBufferAddress();
    valueAddresses[batchIndex] = accumulationVector.getDataBufferAddress();
  }
}