org.apache.arrow.vector.complex.StructVector Java Examples

private static ValueVectorHelper getHelperNull(ValueVector v) {
  if (v instanceof ZeroVector) {
    return new ZeroVectorHelper((ZeroVector) v);
  } else if (v instanceof NullVector) {
    return new NullVectorHelper((NullVector) v);
  } else if (v instanceof UnionVector) {
    return new UnionVectorHelper((UnionVector) v);
  } else if (v instanceof ListVector) {
    return new ListVectorHelper((ListVector) v);
  } else if (v instanceof StructVector) {
    return new StructVectorHelper((StructVector) v);
  } else if (v instanceof NonNullableStructVector) {
    return new NonNullableStructVectorHelper((NonNullableStructVector) v);
  } else if (v instanceof BaseFixedWidthVector) {
    return new FixedWidthVectorHelper<BaseFixedWidthVector>((BaseFixedWidthVector) v);
  } else if (v instanceof BaseVariableWidthVector) {
    return new VariableWidthVectorHelper<BaseVariableWidthVector>((BaseVariableWidthVector) v);
  }

  return null;
}
 

StructCopier(HiveColumnVectorData columnVectorData,
             int ordinalId,
             StructColumnVector inputVector,
             StructVector outputVector, HiveOperatorContextOptions operatorContextOptions) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;

  int fieldCount = inputVector.fields.length;
  int arrowIdx = 0;
  int childPos = ordinalId + 1; // first child is immediately next to struct vector itself

  for (int idx=0; idx<fieldCount; ++idx) {
    if (columnVectorData.isColumnVectorIncluded(childPos)) {
      ValueVector arrowElementVector = outputVector.getVectorById(arrowIdx);
      ColumnVector hiveElementVector = inputVector.fields[idx];
      ORCCopier childCopier = createCopier(columnVectorData, childPos,
        arrowElementVector, hiveElementVector, operatorContextOptions);
      fieldCopiers.add(childCopier);
      arrowIdx++;
    }
    else {
      fieldCopiers.add(new NoOpCopier(null, null));
    }
    childPos += columnVectorData.getTotalVectorCount(childPos);
  }
}
 

/**
 * TODO merge StructVector used by Snowflake timestamp types
 *
 * @param left
 * @param right
 */
private void mergeStructVector(StructVector left, StructVector right) throws SFException
{
  int numOfChildren = left.getChildrenFromFields().size();
  for (int i = 0; i < numOfChildren; i++)
  {
    mergeNonStructVector(left.getChildrenFromFields().get(i),
                         right.getChildrenFromFields().get(i));
  }
  int offset = left.getValueCount();
  for (int i = 0; i < right.getValueCount(); i++)
  {
    if (right.isNull(i))
    {
      left.setNull(offset + i);
    }
  }
  left.setValueCount(offset + right.getValueCount());
}
 

/**
 * Set the vector of Struct and initialize it.
 */
public ArrowFixedSchemaStructMemoryAllocator(
    final StructContainerField schema ,
    final BufferAllocator allocator ,
    final StructVector vector ,
    final int rowCount ) throws IOException {
  this.vector = vector;
  vector.allocateNew();

  loaderMap = new HashMap<String,IMemoryAllocator>();
  for ( String key : schema.getKeys() ) {
    IField childSchema = schema.get( key );
    loaderMap.put( key , ArrowFixedSchemaMemoryAllocatorFactory.getFromStructVector(
        childSchema , key , allocator , vector , rowCount ) );
  }
}
 

@Test
public void T_setString_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new StringObj( "a" ) , 0 );
  column.add( ColumnType.STRING , new StringObj( "b" ) , 1 );
  column.add( ColumnType.STRING , new StringObj( "c" ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeDumpStringColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readText() , null );
}
 

@Test
public void T_setInteger_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.INTEGER , "boolean" );
  column.add( ColumnType.INTEGER , new IntegerObj( 100 ) , 0 );
  column.add( ColumnType.INTEGER , new IntegerObj( 200 ) , 1 );
  column.add( ColumnType.INTEGER , new IntegerObj( 255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  reader.setPosition( 2 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readInteger() , null );
}
 

@Test
public void T_setString_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 4 );

  memoryAllocator.setString( 0 , "a" );
  memoryAllocator.setString( 1 , "b" );
  memoryAllocator.setString( 5 , "c" );
  memoryAllocator.setString( 1000 , "a b c" );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readText() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readText().toString() , "a b c" );
}
 

@Test
public void T_setBytes_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new BytesObj( "a".getBytes() ) , 0 );
  column.add( ColumnType.STRING , new BytesObj( "b".getBytes() ) , 1 );
  column.add( ColumnType.STRING , new BytesObj( "c".getBytes() ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBytesColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByteArray() , null );
}
 

@Test
public void T_setDouble_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.DOUBLE , "boolean" );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)100 ) , 0 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)200 ) , 1 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeDoubleColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readDouble() , null );
}
 

@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}
 

@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
 

public static IMemoryAllocator getFromStructVector( final IField schema , final String columnName , final BufferAllocator allocator , final StructVector vector , final int rowCount ) throws IOException{
  switch( schema.getFieldType() ){
    case UNION:
      return NullMemoryAllocator.INSTANCE;
    case ARRAY:
      return new ArrowFixedSchemaArrayMemoryAllocator( (ArrayContainerField)schema , allocator , vector.addOrGetList( columnName ) , rowCount );
    case MAP:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowFixedSchemaMapMemoryAllocator( (MapContainerField)schema , allocator , mapVector , rowCount );
    case STRUCT:
      StructVector structVector = vector.addOrGetStruct( columnName );
      return new ArrowFixedSchemaStructMemoryAllocator( (StructContainerField)schema , allocator , structVector , rowCount );

    case BOOLEAN:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , columnName ,  allocator , vector , rowCount );
    case BYTE:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , columnName ,  allocator , vector , rowCount );
    case SHORT:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , columnName ,  allocator , vector , rowCount );
    case INTEGER:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , columnName ,  allocator , vector , rowCount );
    case LONG:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , columnName ,  allocator , vector , rowCount );
    case FLOAT:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , columnName ,  allocator , vector , rowCount );
    case DOUBLE:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , columnName ,  allocator , vector , rowCount );
    case STRING:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , columnName ,  allocator , vector , rowCount );
    case BYTES:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , columnName ,  allocator , vector , rowCount );

    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
 

@Test
public void testStructCopier() {
  try (final StructVector s1 = StructVector.empty("s1", allocator);
       final StructVector s2 = StructVector.empty("s2", allocator)) {
    int rowcount = 5000;
    s1.addOrGet("struct_child", FieldType.nullable(Types.MinorType.INT.getType()), IntVector.class);
    s2.addOrGet("struct_child", FieldType.nullable(Types.MinorType.INT.getType()), IntVector.class);
    s1.allocateNew();
    BaseWriter.StructWriter structWriter = new NullableStructWriter(s1);
    IntWriter intWriter = structWriter.integer("struct_child");
    for(int i = 0; i<rowcount; i++) {
      if (i % 10 == 0) {
        continue;
      }
      structWriter.setPosition(i);
      structWriter.start();
      intWriter.writeInt(i);
      structWriter.end();
    }
    s1.setValueCount(rowcount);
    HiveParquetCopier.StructCopier structCopier = new HiveParquetCopier.StructCopier(s1, s2);
    structCopier.copyNonDataBufferRefs(rowcount);
    Assert.assertEquals(rowcount, s2.getValueCount());
    Assert.assertEquals(500, s1.getNullCount());
    Assert.assertEquals(500, s2.getNullCount());
    for(int i=0; i<rowcount; ++i) {
      Assert.assertEquals( i % 10 == 0, s2.isNull(i));
    }
  }
}
 

/**
 * todo append values from the right vector to the left
 *
 * @param left
 * @param right
 */
private void mergeVector(ValueVector left, ValueVector right) throws SFException
{
  if (left instanceof StructVector)
  {
    mergeStructVector((StructVector) left, (StructVector) right);
  }
  else
  {
    mergeNonStructVector(left, right);
  }
}
 

public ArrowFixedSchemaMapMemoryAllocator( final MapContainerField schema , final BufferAllocator allocator , final StructVector vector , final int rowCount ){
  this.allocator = allocator;
  this.vector = vector;
  this.rowCount = rowCount;
  vector.allocateNew();
  childSchema = schema.getField();
}
 

public static IMemoryAllocator getFromListVector( final IField schema , final String columnName , final BufferAllocator allocator , final ListVector vector , final int rowCount ) throws IOException{
  switch( schema.getFieldType() ){
    case UNION:
      return NullMemoryAllocator.INSTANCE;
    case ARRAY:
      AddOrGetResult<ListVector> listVector =  vector.addOrGetVector( new FieldType( true , ArrowType.List.INSTANCE , null , null ) );
      return new ArrowFixedSchemaArrayMemoryAllocator( (ArrayContainerField)schema , allocator , listVector.getVector() , rowCount );
    case MAP:
      AddOrGetResult<StructVector> mapVector =  vector.addOrGetVector( new FieldType( true , ArrowType.Struct.INSTANCE , null , null ) );
      return new ArrowFixedSchemaMapMemoryAllocator( (MapContainerField)schema , allocator , mapVector.getVector() , rowCount );
    case STRUCT:
      AddOrGetResult<StructVector> structVector =  vector.addOrGetVector( new FieldType( true , ArrowType.Struct.INSTANCE , null , null ) );
      return new ArrowFixedSchemaStructMemoryAllocator( (StructContainerField)schema , allocator , structVector.getVector() , rowCount );

    case BOOLEAN:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.BOOLEAN , columnName ,  allocator , vector , rowCount );
    case BYTE:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.BYTE , columnName ,  allocator , vector , rowCount );
    case SHORT:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.SHORT , columnName ,  allocator , vector , rowCount );
    case INTEGER:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.INTEGER , columnName ,  allocator , vector , rowCount );
    case LONG:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.LONG , columnName ,  allocator , vector , rowCount );
    case FLOAT:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.FLOAT , columnName ,  allocator , vector , rowCount );
    case DOUBLE:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.DOUBLE , columnName ,  allocator , vector , rowCount );
    case STRING:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.STRING , columnName ,  allocator , vector , rowCount );
    case BYTES:
      return ArrowMemoryAllocatorFactory.getFromListVector( ColumnType.BYTES , columnName ,  allocator , vector , rowCount );

    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
 

@Test
public void T_setFloat_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.FLOAT , "boolean" );
  column.add( ColumnType.FLOAT , new FloatObj( (float)100 ) , 0 );
  column.add( ColumnType.FLOAT , new FloatObj( (float)200 ) , 1 );
  column.add( ColumnType.FLOAT , new FloatObj( (float)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeFloatColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readFloat().floatValue() , (float)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readFloat().floatValue() , (float)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readFloat().floatValue() , (float)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readFloat() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readFloat() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readFloat() , null );
}
 

@Test
public void T_setFloat_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , "target" , allocator , parent , 4 );

  memoryAllocator.setFloat( 0 , (float)0.1 );
  memoryAllocator.setFloat( 1 , (float)0.2 );
  memoryAllocator.setFloat( 5 , (float)0.255 );
  memoryAllocator.setFloat( 1000 , (float)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readFloat().floatValue() , (float)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readFloat().floatValue() , (float)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readFloat().floatValue() , (float)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readFloat() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readFloat().floatValue() , (float)0.1 );
}
 

public ArrowStructColumn( final String columnName , final StructVector vector ){
  this.columnName = columnName;

  List<FieldVector> vectorList = vector.getChildrenFromFields();
  spread = new Spread();
  for( FieldVector v : vectorList ){
    spread.addColumn( ArrowColumnFactory.convert( v.getField().getName() , v ) );
  }
  spread.setRowCount( vector.getValueCount() );
}
 

public ArrowFixedSchemaStructMemoryAllocator( final StructContainerField schema , final BufferAllocator allocator , final StructVector vector , final int rowCount ) throws IOException{
  this.vector = vector;
  vector.allocateNew();

  loaderMap = new HashMap<String,IMemoryAllocator>();
  for( String key : schema.getKeys() ){
    IField childSchema = schema.get( key );
    loaderMap.put( key , ArrowFixedSchemaMemoryAllocatorFactory.getFromStructVector( childSchema , key , allocator , vector , rowCount ) );
  }
}
 

public DynamicArrowLoader( final IRootMemoryAllocator rootMemoryAllocator , final MDSReader reader , final BufferAllocator allocator ){
  this.reader = reader;
  this.allocator = allocator;
  this.rootMemoryAllocator = rootMemoryAllocator;
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  rootVector = new StructVector( "root" , allocator , new FieldType( true , Struct.INSTANCE , null , null ) , callBack );
}
 

@Test
public void T_setShort_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 1001 );

  memoryAllocator.setShort( 0 , (short)100 );
  memoryAllocator.setShort( 1 , (short)200 );
  memoryAllocator.setShort( 5 , (short)255 );
  memoryAllocator.setShort( 1000 , (short)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readShort() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( (short)( reader.readShort() ) , (short)10 );
}
 

public DirectArrowLoader( final IRootMemoryAllocator rootMemoryAllocator , final MDSReader reader , final BufferAllocator allocator ){
  this.reader = reader;
  this.allocator = allocator;
  this.rootMemoryAllocator = rootMemoryAllocator;
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  rootVector = new StructVector( "root" , allocator , new FieldType( true , Struct.INSTANCE , null , null ) , callBack );
}
 

@Test
public void T_setShort_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.SHORT , "boolean" );
  column.add( ColumnType.SHORT , new ShortObj( (short)100 ) , 0 );
  column.add( ColumnType.SHORT , new ShortObj( (short)200 ) , 1 );
  column.add( ColumnType.SHORT , new ShortObj( (short)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readShort() , null );
}
 

@Test
public void T_setByte_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 1001 );

  memoryAllocator.setByte( 0 , (byte)100 );
  memoryAllocator.setByte( 1 , (byte)200 );
  memoryAllocator.setByte( 5 , (byte)255 );
  memoryAllocator.setByte( 1000 , (byte)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readByte() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readByte().byteValue() , (byte)10 );
}
 

@NotNull
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private static ArrowVectorAccessor getPlainVectorAccessor(FieldVector vector) {
  if (vector instanceof BitVector) {
    return new BooleanAccessor((BitVector) vector);
  } else if (vector instanceof IntVector) {
    return new IntAccessor((IntVector) vector);
  } else if (vector instanceof BigIntVector) {
    return new LongAccessor((BigIntVector) vector);
  } else if (vector instanceof Float4Vector) {
    return new FloatAccessor((Float4Vector) vector);
  } else if (vector instanceof Float8Vector) {
    return new DoubleAccessor((Float8Vector) vector);
  } else if (vector instanceof IcebergArrowVectors.DecimalArrowVector) {
    return new DecimalAccessor((IcebergArrowVectors.DecimalArrowVector) vector);
  } else if (vector instanceof IcebergArrowVectors.VarcharArrowVector) {
    return new StringAccessor((IcebergArrowVectors.VarcharArrowVector) vector);
  } else if (vector instanceof VarBinaryVector) {
    return new BinaryAccessor((VarBinaryVector) vector);
  } else if (vector instanceof DateDayVector) {
    return new DateAccessor((DateDayVector) vector);
  } else if (vector instanceof TimeStampMicroTZVector) {
    return new TimestampAccessor((TimeStampMicroTZVector) vector);
  } else if (vector instanceof ListVector) {
    ListVector listVector = (ListVector) vector;
    return new ArrayAccessor(listVector);
  } else if (vector instanceof StructVector) {
    StructVector structVector = (StructVector) vector;
    return new StructAccessor(structVector);
  }
  throw new UnsupportedOperationException("Unsupported vector: " + vector.getClass());
}
 

@Test
public void T_setByte_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BYTE , "boolean" );
  column.add( ColumnType.BYTE , new ByteObj( (byte)100 ) , 0 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)200 ) , 1 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByte() , null );
}
 

@Override
public void doWrite(T in, int ordinal) {
	RowData row;
	if (isNullAt(in, ordinal)) {
		row = nullRow;
		((StructVector) getValueVector()).setNull(getCount());
	} else {
		row = readRow(in, ordinal);
		((StructVector) getValueVector()).setIndexDefined(getCount());
	}
	for (int i = 0; i < fieldsWriters.length; i++) {
		fieldsWriters[i].write(row, i);
	}
}
 

@Test
public void T_setByte_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BYTE , "boolean" );
  column.add( ColumnType.BYTE , new ByteObj( (byte)100 ) , 0 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)200 ) , 1 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByte() , null );
}
 

@Test
public void T_setShort_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.SHORT , "boolean" );
  column.add( ColumnType.SHORT , new ShortObj( (short)100 ) , 0 );
  column.add( ColumnType.SHORT , new ShortObj( (short)200 ) , 1 );
  column.add( ColumnType.SHORT , new ShortObj( (short)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readShort() , null );
}