org.apache.arrow.vector.BigIntVector Java Examples

@Test
public void testGetDefaultValueIfNullMethod() throws Exception
{
    Schema inputSchema = SchemaBuilder.newBuilder()
            .addField("input", Types.MinorType.BIGINT.getType())
            .build();
    Schema outputSchema = SchemaBuilder.newBuilder()
            .addField("output", Types.MinorType.BIGINT.getType())
            .build();

    Block inputRecords = allocator.createBlock(inputSchema);
    inputRecords.setRowCount(2);
    BigIntVector fieldVector = (BigIntVector) inputRecords.getFieldVector("input");
    fieldVector.setSafe(0, 123l);
    fieldVector.setNull(1);

    UserDefinedFunctionResponse response = runAndAssertSerialization(inputRecords, outputSchema, "get_default_value_if_null");

    Block outputRecords = response.getRecords();
    assertEquals(2, outputRecords.getRowCount());
    FieldReader fieldReader = outputRecords.getFieldReader("output");
    ArrowValueProjector arrowValueProjector = ProjectorUtils.createArrowValueProjector(fieldReader);
    assertEquals(exampleUserDefinedFunctionHandler.get_default_value_if_null(123l), arrowValueProjector.project(0));
    assertEquals(exampleUserDefinedFunctionHandler.get_default_value_if_null(null), arrowValueProjector.project(1));
}
 

private Timestamp getTimestamp(int index, TimeZone tz) throws SFException
{
  long epoch = epochs.getDataBuffer().getLong(index * BigIntVector.TYPE_WIDTH);
  int timeZoneIndex = timeZoneIndices.getDataBuffer().getInt(index * IntVector.TYPE_WIDTH);

  Timestamp ts = ArrowResultUtil.toJavaTimestamp(epoch, context.getScale(columnIndex));

  if (context.getResultVersion() > 0)
  {
    timeZone = SFTimestamp.convertTimezoneIndexToTimeZone(timeZoneIndex);
  }
  else
  {
    timeZone = TimeZone.getTimeZone("UTC");
  }

  Timestamp adjustedTimestamp = ResultUtil.adjustTimestamp(ts);

  return adjustedTimestamp;
}
 

@Test
public void testNullableAgg() throws Exception {
  final List<QueryDataBatch> result = testSqlWithResults(
      "select sum(a) as total_sum from dfs.\"/tmp/parquet_with_nulls_should_sum_100000_nulls_first.parquet\"");
  assertEquals("Only expected one batch with data, and then the empty finishing batch.", 2, result.size());
  final RecordBatchLoader loader = new RecordBatchLoader(getSabotContext().getAllocator());

  final QueryDataBatch b = result.get(0);
  loader.load(b.getHeader().getDef(), b.getData());

  final VectorWrapper vw = loader.getValueAccessorById(
      BigIntVector.class,
      loader.getValueVectorId(SchemaPath.getCompoundPath("total_sum")).getFieldIds()
  );
  assertEquals(4999950000l, vw.getValueVector().getObject(0));
  b.release();
  loader.clear();
}
 

/**
 * Method is invoked when we have a straight aggregate (no group by expression) and our input is empty.
 * In this case we construct an outgoing batch with record count as 1. For the nullable vectors we don't set anything
 * as we want the output to be NULL. For the required vectors (only for count()) we set the value to be zero since
 * we don't zero out our buffers initially while allocating them.
 */
private void constructSpecialBatch() {
  outgoing.allocateNew();
  List<NamedExpression> exprs = config.getAggrExprs();
  if(outgoing.getNumberOfColumns() != exprs.size()){
    throw new IllegalStateException();
  }
  int exprIndex = 0;
  for (final VectorWrapper<?> vw: outgoing) {
    final ValueVector vv = vw.getValueVector();
    if (!exprs.isEmpty() && isCount(exprs.get(exprIndex))) {
      ((BigIntVector) vv).setSafe(0, 0);
    }
    vv.setValueCount(SPECIAL_BATCH_COUNT);
    exprIndex++;
  }
  outgoing.setRecordCount(SPECIAL_BATCH_COUNT);
}
 

private Timestamp getTimestamp(int index, TimeZone tz, boolean fromToString) throws SFException
{
  long epoch = epochs.getDataBuffer().getLong(index * BigIntVector.TYPE_WIDTH);
  int fraction = fractions.getDataBuffer().getInt(index * IntVector.TYPE_WIDTH);

  if (ArrowResultUtil.isTimestampOverflow(epoch))
  {
    if (fromToString)
    {
      throw new TimestampOperationNotAvailableException(epoch, fraction);
    }
    else
    {
      return null;
    }
  }

  Timestamp ts = ArrowResultUtil.createTimestamp(epoch, fraction, false);

  Timestamp adjustedTimestamp = ResultUtil.adjustTimestamp(ts);

  return adjustedTimestamp;
}
 

private Pair<BigIntVector, ResultVerifier> testBigIntVector() {
  final String colName = "colBigInt";
  final List<Long> values = asList(null, 50L, -2000L, 327345234234L, 0L);

  BigIntVector valuesVector = new BigIntVector(colName, allocator);
  valuesVector.allocateNew(values.size());
  for (int i = 0; i < values.size(); i++) {
    if (values.get(i) == null) {
      valuesVector.setNull(i);
    } else {
      valuesVector.set(i, values.get(i));
    }
  }

  ResultVerifier verifier = new ResultVerifier() {
    @Override
    public void verify(DataPOJO output) {
      verifyIntValues(values, output, colName);
    }
  };

  return Pair.of(valuesVector, verifier);
}
 

@Test
public void testRead() throws Exception {
  List<QueryDataBatch> results = testSqlWithResults(
      String.format("SELECT count(*) FROM dfs.\"%s\"", dataFile.getPath()));

  RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());

  for(QueryDataBatch batch : results) {
    batchLoader.load(batch.getHeader().getDef(), batch.getData());

    if (batchLoader.getRecordCount() <= 0) {
      continue;
    }

    BigIntVector countV = batchLoader.getValueAccessorById(BigIntVector.class, 0).getValueVector();
    assertTrue("Total of "+ NUM_RECORDS + " records expected in count", countV.get(0) == NUM_RECORDS);

    batchLoader.clear();
    batch.release();
  }
}
 

@Test
public void testGetBooleanNoScale() throws SFException
{
  Map<String, String> customFieldMeta = new HashMap<>();
  customFieldMeta.put("logicalType", "FIXED");
  customFieldMeta.put("precision", "10");
  customFieldMeta.put("scale", "0");

  FieldType fieldType = new FieldType(true,
                                      Types.MinorType.BIGINT.getType(),
                                      null, customFieldMeta);

  BigIntVector vector = new BigIntVector("col_one", fieldType, allocator);
  vector.setSafe(0, 0);
  vector.setSafe(1, 1);
  vector.setNull(2);
  vector.setSafe(3, 5);

  ArrowVectorConverter converter = new BigIntToFixedConverter(vector, 0, this);

  assertThat(false, is(converter.toBoolean(0)));
  assertThat(true, is(converter.toBoolean(1)));
  assertThat(false, is(converter.toBoolean(2)));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toBoolean(3));

  vector.close();
}
 

@Test
public void fixedOnlyWithoutNull(){
  try(IntVector col1 = new IntVector("col1", allocator);
      BigIntVector col2 = new BigIntVector("col2", allocator);
      DecimalVector col3 = new DecimalVector("col3", allocator, 30, 0);
      BitVector col4 = new BitVector("col4", allocator)){

    PivotDef pivot = PivotBuilder.getBlockDefinition(
      new FieldVectorPair(col1, col1),
      new FieldVectorPair(col2, col2),
      new FieldVectorPair(col3, col3),
      new FieldVectorPair(col4, col4)
    );
    Integer col1Val[] = populate4ByteValuesWithoutNull(col1, 4096);
    Long col2Val[] = populate8ByteValuesWithoutNull(col2, 4096);
    BigDecimal col3Val[] = populate16ByteValuesWithoutNull(col3, 4096);
    Boolean col4Val[] = populateBooleanValuesWithoutNull(col4, 4096);

    assertEquals(32, pivot.getBlockWidth());

    try(FixedBlockVector fixed = new FixedBlockVector(allocator, pivot.getBlockWidth(), 4096, true);
        VariableBlockVector variable = new VariableBlockVector(allocator, pivot.getVariableCount(), 4096 * 10, true)){
      fixedOnlyHelper(pivot, fixed, variable, 0, 4096, false, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 0, 128, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 0, 17, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 0, 76, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 5, 39, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 5, 189, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 5, 123, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 0, 1023, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 1023, 1023, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 2046, 1023, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 3069, 1023, true, col1Val, col2Val, col3Val, col4Val);
      fixedOnlyHelper(pivot, fixed, variable, 4092, 4, true, col1Val, col2Val, col3Val, col4Val);
    }
  }
}
 

@Test
public void testGetBooleanWithScale() throws SFException
{
  Map<String, String> customFieldMeta = new HashMap<>();
  customFieldMeta.put("logicalType", "FIXED");
  customFieldMeta.put("precision", "10");
  customFieldMeta.put("scale", "3");

  FieldType fieldType = new FieldType(true,
                                      Types.MinorType.BIGINT.getType(),
                                      null, customFieldMeta);

  BigIntVector vector = new BigIntVector("col_one", fieldType, allocator);
  vector.setSafe(0, 0);
  vector.setSafe(1, 1);
  vector.setNull(2);
  vector.setSafe(3, 5);

  final ArrowVectorConverter converter = new BigIntToScaledFixedConverter(vector, 0, this, 3);

  assertThat(false, is(converter.toBoolean(0)));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toBoolean(3));
  assertThat(false, is(converter.toBoolean(2)));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toBoolean(3));

  vector.close();
}
 

@Test
public void testMemoryAllocation() throws Exception {

  try (BufferAllocator allocator = allocatorRule.newAllocator("test-greedy-memory-allocation", 0, Long.MAX_VALUE)) {

    BigIntVector in1 = new BigIntVector("in1", allocator);
    BigIntVector in2 = new BigIntVector("in2", allocator);
    BigIntVector in3 = new BigIntVector("in3", allocator);
    BigIntVector in4 = new BigIntVector("in4", allocator);

    BigIntVector out1 = new BigIntVector("in1-sum", allocator);
    BigIntVector out2 = new BigIntVector("in2-sum", allocator);
    BigIntVector out3 = new BigIntVector("in3-sum", allocator);
    BigIntVector out4 = new BigIntVector("in4-sum", allocator);

    final SumAccumulators.BigIntSumAccumulator ac1 = new SumAccumulators.BigIntSumAccumulator(in1, out1, out1, MAX_VALUES_PER_BATCH, allocator);
    final SumAccumulators.BigIntSumAccumulator ac2 = new SumAccumulators.BigIntSumAccumulator(in2, out2, out2, MAX_VALUES_PER_BATCH, allocator);
    final SumAccumulators.BigIntSumAccumulator ac3 = new SumAccumulators.BigIntSumAccumulator(in3, out3, out3, MAX_VALUES_PER_BATCH, allocator);
    final SumAccumulators.BigIntSumAccumulator ac4 = new SumAccumulators.BigIntSumAccumulator(in4, out4, out4, MAX_VALUES_PER_BATCH, allocator);

    final AccumulatorSet accumulatorSet = new AccumulatorSet(JOINT_ALLOCATION_MIN, JOINT_ALLOCATION_MAX, allocator, ac1, ac2, ac3, ac4);

    accumulatorSet.addBatch();
    Map<Integer, List<List<Integer>>> allocationMapping = accumulatorSet.getMapping();
    assertEquals(1, allocationMapping.size());
    assertEquals(3, allocationMapping.keySet().iterator().next().intValue()); // 32KB bucket
    List<List<Integer>> ranges = allocationMapping.get(3);
    assertEquals(1, ranges.size());

    assertArrayEquals(new Integer[] {0, 1, 2, 3}, ranges.get(0).toArray());

    //In each accumulator, 4 bytes for value and 1 bit for validity.
    final int allocatedMemory = (round8(8 * MAX_VALUES_PER_BATCH) + round8(getValidityBufferSizeFromCount(MAX_VALUES_PER_BATCH))) * 4;
    assertEquals(roundPower2(allocatedMemory), allocator.getAllocatedMemory());
    accumulatorSet.close();
  }
}
 

private void testCTASQueryHelper(String tableName, String testQuery, int expectedOutputCount) throws Exception {
  try {
    List<QueryDataBatch> results = testSqlWithResults(testQuery);

    RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());

    int recordsWritten = 0;
    for (QueryDataBatch batch : results) {
      batchLoader.load(batch.getHeader().getDef(), batch.getData());

      if (batchLoader.getRecordCount() <= 0) {
        continue;
      }

      BigIntVector recordWrittenV = (BigIntVector) batchLoader.getValueAccessorById(BigIntVector.class, 1).getValueVector();

      for (int i = 0; i < batchLoader.getRecordCount(); i++) {
        recordsWritten += recordWrittenV.get(i);
      }

      batchLoader.clear();
      batch.release();
    }

    assertEquals(expectedOutputCount, recordsWritten);
  } finally {
    try {
      Path path = new Path(getDfsTestTmpSchemaLocation(), tableName);
      if (fs.exists(path)) {
        fs.delete(path, true);
      }
    } catch (Exception e) {
      // ignore exceptions.
      logger.warn("Failed to delete the table [{}, {}] created as part of the test",
          getDfsTestTmpSchemaLocation(), tableName);
    }
  }
}
 

private ValueVector getValueVector(int index) {
  Object tmp = (htContainer).getValueAccessorById(BigIntVector.class, index).getValueVector();
  if (tmp != null) {
    BigIntVector vv0 = ((BigIntVector) tmp);
    return vv0;
  }
  return null;
}
 

@Test
public void testInvalidConversion()
{
  // try convert to int/long/byte/short with scale > 0
  Map<String, String> customFieldMeta = new HashMap<>();
  customFieldMeta.put("logicalType", "FIXED");
  customFieldMeta.put("precision", "10");
  customFieldMeta.put("scale", "3");

  FieldType fieldType = new FieldType(true,
                                      Types.MinorType.BIGINT.getType(),
                                      null, customFieldMeta);

  BigIntVector vector = new BigIntVector("col_one", fieldType, allocator);
  vector.setSafe(0, 123456789L);

  final ArrowVectorConverter converter = new BigIntToScaledFixedConverter(vector, 0, this, 3);

  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toBoolean(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toLong(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toInt(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toShort(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toByte(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toDate(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toTime(0));
  TestUtil.assertSFException(invalidConversionErrorCode,
                             () -> converter.toTimestamp(0, TimeZone.getDefault()));
  vector.clear();
}
 

private static VectorContainer buildLongGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(64, true), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final BigIntVector longVector = input.addOrGet(field);
  longVector.allocateNew();
  SortedSet<Long> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToLong(i));
    }
  }
  if (existingDict != null) {
    final BigIntVector existingDictValues = existingDict.getValueAccessorById(BigIntVector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.get(i));
    }
  }
  final Iterator<Long> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    longVector.setSafe(recordCount++, iter.next());
  }
  longVector.setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

@Test
public void T_convert_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  BigIntVector vector = new BigIntVector( "test" , allocator );
  vector.allocateNew();
  vector.setSafe( 0 , (long)0 );  
  vector.setSafe( 1 , (long)1 );  
  vector.setSafe( 2 , (long)0 );  
  vector.setNull( 3 );  
  vector.setSafe( 4 , (long)1 );  
  vector.setSafe( 5 , (long)1 );  
  vector.setSafe( 6 , (long)1 );  
  vector.setNull( 7 );  
  vector.setValueCount( 8 );

  IColumn column = ArrowColumnFactory.convert( "test" , vector );
  assertEquals( column.getColumnName() , "test" );
  assertEquals( column.size() , 8 );
  assertTrue( ( column.getColumnType() == ColumnType.LONG ) );
  assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getLong() , (long)0  );
  assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getLong() , (long)1  );
  assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getLong() , (long)0  );
  assertEquals( column.get(3).getRow() , null  );
  assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getLong() , (long)1 );
  assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getLong() , (long)1 );
  assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getLong() , (long)1 );
  assertEquals( column.get(7).getRow() , null  );
}
 

@NotNull
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private static ArrowVectorAccessor getPlainVectorAccessor(FieldVector vector) {
  if (vector instanceof BitVector) {
    return new BooleanAccessor((BitVector) vector);
  } else if (vector instanceof IntVector) {
    return new IntAccessor((IntVector) vector);
  } else if (vector instanceof BigIntVector) {
    return new LongAccessor((BigIntVector) vector);
  } else if (vector instanceof Float4Vector) {
    return new FloatAccessor((Float4Vector) vector);
  } else if (vector instanceof Float8Vector) {
    return new DoubleAccessor((Float8Vector) vector);
  } else if (vector instanceof IcebergArrowVectors.DecimalArrowVector) {
    return new DecimalAccessor((IcebergArrowVectors.DecimalArrowVector) vector);
  } else if (vector instanceof IcebergArrowVectors.VarcharArrowVector) {
    return new StringAccessor((IcebergArrowVectors.VarcharArrowVector) vector);
  } else if (vector instanceof VarBinaryVector) {
    return new BinaryAccessor((VarBinaryVector) vector);
  } else if (vector instanceof DateDayVector) {
    return new DateAccessor((DateDayVector) vector);
  } else if (vector instanceof TimeStampMicroTZVector) {
    return new TimestampAccessor((TimeStampMicroTZVector) vector);
  } else if (vector instanceof ListVector) {
    ListVector listVector = (ListVector) vector;
    return new ArrayAccessor(listVector);
  } else if (vector instanceof StructVector) {
    StructVector structVector = (StructVector) vector;
    return new StructAccessor(structVector);
  }
  throw new UnsupportedOperationException("Unsupported vector: " + vector.getClass());
}
 

@Test
public void bigintRoundtrip() throws Exception {
  final int count = 1024;
  try (
    BigIntVector in = new BigIntVector("in", allocator);
    BigIntVector out = new BigIntVector("out", allocator);
  ) {

    in.allocateNew(count);

    for (int i = 0; i < count; i++) {
      if (i % 5 == 0) {
        in.setSafe(i, i);
      }
    }
    in.setValueCount(count);

    final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out));
    try (
      final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth());
      final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount());
    ) {
      fbv.ensureAvailableBlocks(count);
      Pivots.pivot(pivot, count, fbv, vbv);

      ValueVector[] ins = new ValueVector[]{in};
      ValueVector[] outs = new ValueVector[]{out};
      unpivotHelper(pivot, fbv, vbv, ins, outs, 0, count);
      unpivotHelper(pivot, fbv, vbv, ins, outs, 0, 100);
      unpivotHelper(pivot, fbv, vbv, ins, outs, 100, 924);
    }
  }
}
 

private AccumulatorSet createAccumulator(IntVector in1,
                                         final BufferAllocator allocator) {
  /* SUM Accumulator */
  BigIntVector in1SumOutputVector = new BigIntVector("int-sum", allocator);
  final SumAccumulators.IntSumAccumulator in1SumAccum =
    new SumAccumulators.IntSumAccumulator(in1, in1SumOutputVector, in1SumOutputVector, MAX_VALUES_PER_BATCH, allocator);

  /* Min Accumulator */
  IntVector in1MaxOutputVector = new IntVector("int-max", allocator);
  final MaxAccumulators.IntMaxAccumulator in1MaxAccum =
    new MaxAccumulators.IntMaxAccumulator(in1, in1MaxOutputVector, in1MaxOutputVector, MAX_VALUES_PER_BATCH, allocator);

  return new AccumulatorSet(4*1024, 64*1024, allocator, in1SumAccum, in1MaxAccum);
}
 

@Override
public void doWrite(T in, int ordinal) {
	if (isNullAt(in, ordinal)) {
		((BigIntVector) getValueVector()).setNull(getCount());
	} else {
		((BigIntVector) getValueVector()).setSafe(getCount(), readLong(in, ordinal));
	}
}
 

@Test
public void fixedVariable(){
  try(IntVector col1 = new IntVector("col1", allocator);
      BigIntVector col2 = new BigIntVector("col2", allocator);
      VarCharVector col3 = new VarCharVector("col3", allocator);){

    PivotDef pivot = PivotBuilder.getBlockDefinition(
        new FieldVectorPair(col1, col1),
        new FieldVectorPair(col2, col2),
        new FieldVectorPair(col3, col3)
    );
    Integer col1Val[] = populate4ByteValues(col1, 4096);
    Long col2Val[] = populate8ByteValues(col2, 4096);
    String col3Val[] = populateVarCharValues(col3, 4096);

    assertEquals(20, pivot.getBlockWidth());

    try(FixedBlockVector fixed = new FixedBlockVector(allocator, pivot.getBlockWidth(), 4096, true);
        VariableBlockVector variable = new VariableBlockVector(allocator, pivot.getVariableCount(), 4096 * 2, true)){
      fixedVariableHelper(pivot, fixed, variable, 0, 4096, false, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 0, 128, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 0, 17, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 0, 76, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 5, 39, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 5, 189, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 5, 123, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 0, 1023, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 1023, 1023, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 2046, 1023, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 3069, 1023, true, col1Val, col2Val, col3Val);
      fixedVariableHelper(pivot, fixed, variable, 4092, 4, true, col1Val, col2Val, col3Val);
    }
  }
}
 

public TwoFieldStructToTimestampLTZConverter(ValueVector fieldVector, int columnIndex, DataConversionContext context)
{
  super(SnowflakeType.TIMESTAMP_LTZ.name(), fieldVector, columnIndex, context);
  structVector = (StructVector) fieldVector;
  epochs = structVector.getChild(FIELD_NAME_EPOCH, BigIntVector.class);
  fractions = structVector.getChild(FIELD_NAME_FRACTION, IntVector.class);
}
 

static Long[] populate8ByteValuesWithoutNull(BigIntVector vector, int size){
  vector.allocateNew();
  Long values[] = new Long[size];
  for(int i = 0; i < values.length; i++){
    values[i] = RAND.nextLong();
    vector.setSafe(i, values[i]);
  }
  vector.setValueCount(values.length);
  return values;
}
 

@Test
public void bigintRoundtrip(){
  final int count = 1024;
  try(
      BigIntVector in = new BigIntVector("in", allocator);
      BigIntVector out = new BigIntVector("out", allocator);
      ){

    in.allocateNew(count);

    for(int i = 0; i < count; i++){
      if(i % 5 == 0){
        in.setSafe(i, i);
      }
    }
    in.setValueCount(count);

    List<FieldBufferCopier> copiers = FieldBufferCopier.getCopiers(ImmutableList.<FieldVector>of(in), ImmutableList.<FieldVector>of(out));
    try(
        final SelectionVector2 sv2 = new SelectionVector2(allocator);
        ){

      sv2.allocateNew(count);
      // create full sv2.
      int x = 0;
      for(long mem = sv2.memoryAddress(); mem < sv2.memoryAddress() + count * 2; mem+=2){
        PlatformDependent.putShort(mem, (short) (char) x);
        x++;
      }
      sv2.setRecordCount(count);
      copy(copiers, sv2);

      out.setValueCount(count);
      for(int i =0; i < count; i++){
        assertEquals(in.getObject(i), out.getObject(i));
      }
    }
  }
}
 

@Test
public void bigintAppend(){
  final int count = 1024;
  try(
    BigIntVector in = new BigIntVector("in", allocator);
    BigIntVector out = new BigIntVector("out", allocator);
  ){

    in.allocateNew(count);

    for(int i = 0; i < count; i++){
      if(i % 5 == 0){
        in.setSafe(i, i);
      }
    }
    in.setValueCount(count);

    List<FieldBufferCopier> copiers = FieldBufferCopier.getCopiers(ImmutableList.<FieldVector>of(in), ImmutableList.<FieldVector>of(out));
    try(
      final SelectionVector2 sv2 = new SelectionVector2(allocator);
    ){

      sv2.allocateNew(count / 2);
      // set alternate elements.
      int x = 0;
      for(long mem = sv2.memoryAddress(); mem < sv2.memoryAddress() + count; mem+=2){
        PlatformDependent.putShort(mem, (short) (char) x);
        x += 2;
      }
      sv2.setRecordCount(count/2);
      append(copiers, sv2);

      out.setValueCount(count / 2);
      for(int i =0; i < count / 2; i++){
        assertEquals(in.getObject(i * 2), out.getObject(i));
      }
    }
  }
}
 

public BigIntToFixedConverter(ValueVector fieldVector, int columnIndex, DataConversionContext context)
{
  super(String.format("%s(%s,%s)", SnowflakeType.FIXED,
                      fieldVector.getField().getMetadata().get("precision"),
                      fieldVector.getField().getMetadata().get("scale")),
        fieldVector,
        columnIndex,
        context);
  this.bigIntVector = (BigIntVector) fieldVector;
}
 

@Override
public byte[] toBytes(int index)
{
  if (isNull(index))
  {
    return null;
  }
  else
  {
    byteBuf.putLong(0, bigIntVector.getDataBuffer().getLong(index * BigIntVector.TYPE_WIDTH));
    return byteBuf.array();
  }
}
 

public TwoFieldStructToTimestampTZConverter(ValueVector fieldVector, int columnIndex, DataConversionContext context)
{
  super(SnowflakeType.TIMESTAMP_LTZ.name(), fieldVector, columnIndex, context);
  structVector = (StructVector) fieldVector;
  epochs = structVector.getChild(FIELD_NAME_EPOCH, BigIntVector.class);
  timeZoneIndices = structVector.getChild(FIELD_NAME_TIME_ZONE_INDEX, IntVector.class);
}
 

private Timestamp getTimestamp(int index, TimeZone tz, boolean fromToString) throws SFException
{
  long epoch = epochs.getDataBuffer().getLong(index * BigIntVector.TYPE_WIDTH);
  int fraction = fractions.getDataBuffer().getInt(index * IntVector.TYPE_WIDTH);

  if (ArrowResultUtil.isTimestampOverflow(epoch))
  {
    if (fromToString)
    {
      throw new TimestampOperationNotAvailableException(epoch, fraction);
    }
    else
    {
      return null;
    }
  }
  Timestamp ts = ArrowResultUtil.createTimestamp(epoch, fraction, this.treatNTZasUTC);

  // Note: honorClientTZForTimestampNTZ is not enabled for toString method
  // If JDBC_TREAT_TIMESTAMP_NTZ_AS_UTC=false, default behavior is to honor
  // client timezone for NTZ time. Move NTZ timestamp offset to correspond to
  // client's timezone
  if (!this.treatNTZasUTC && !fromToString && context.getHonorClientTZForTimestampNTZ())
  {
    ts = ArrowResultUtil.moveToTimeZone(ts, NTZ, tz);
  }
  Timestamp adjustedTimestamp = ResultUtil.adjustTimestamp(ts);
  return adjustedTimestamp;
}
 

private Timestamp getTimestamp(int index, TimeZone tz, boolean fromToString) throws SFException
{
  long epoch = epochs.getDataBuffer().getLong(index * BigIntVector.TYPE_WIDTH);
  int fraction = fractions.getDataBuffer().getInt(index * IntVector.TYPE_WIDTH);
  int timeZoneIndex = timeZoneIndices.getDataBuffer().getInt(index * IntVector.TYPE_WIDTH);

  if (ArrowResultUtil.isTimestampOverflow(epoch))
  {
    if (fromToString)
    {
      throw new TimestampOperationNotAvailableException(epoch, fraction);
    }
    else
    {
      return null;
    }
  }
  Timestamp ts = ArrowResultUtil.createTimestamp(epoch, fraction, false);

  if (context.getResultVersion() > 0)
  {
    timeZone = SFTimestamp.convertTimezoneIndexToTimeZone(timeZoneIndex);
  }
  else
  {
    timeZone = TimeZone.getTimeZone("UTC");
  }

  Timestamp adjustedTimestamp = ResultUtil.adjustTimestamp(ts);

  return adjustedTimestamp;
}