org.apache.arrow.vector.types.pojo.ArrowType.Struct Java Examples

@Override
public String visit(Struct type) {
  StringBuilder sb = new StringBuilder();
  if(includeName){
    sb.append(field.getName());
    sb.append("::");
  }

  sb.append("struct<");
  boolean first = true;
  for (Field f : field.getChildren()) {
    if (first) {
      first = false;
    } else {
      sb.append(", ");
    }
    sb.append(describe(f, true));
  }
  sb.append(">");
  return sb.toString();
}
 

@Test
public void T_setInteger_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 1001 );

  memoryAllocator.setInteger( 0 , 100 );
  memoryAllocator.setInteger( 1 , 200 );
  memoryAllocator.setInteger( 5 , 255 );
  memoryAllocator.setInteger( 1000 , 10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readInteger() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readInteger().intValue() , 10 );
}
 

@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
 

@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}
 

@Test
public void T_setDouble_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.DOUBLE , "boolean" );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)100 ) , 0 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)200 ) , 1 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeDoubleColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readDouble() , null );
}
 

@Test
public void T_setBoolean_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 4 );

  memoryAllocator.setBoolean( 0 , true );
  memoryAllocator.setBoolean( 1 , false );
  memoryAllocator.setBoolean( 5 , true );
  memoryAllocator.setBoolean( 1000 , true );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readBoolean() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readBoolean().booleanValue() , true );
}
 

/**
 * FileReader and Arrow memory allocators are set and initialized.
 */
public DirectArrowLoader(
    final IRootMemoryAllocator rootMemoryAllocator ,
    final YosegiReader reader ,
    final BufferAllocator allocator ) {
  this.reader = reader;
  this.allocator = allocator;
  this.rootMemoryAllocator = rootMemoryAllocator;
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  rootVector = new StructVector(
      "root" , allocator , new FieldType( true , Struct.INSTANCE , null , null ) , callBack );
}
 

@Test
public void T_setString_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 4 );

  memoryAllocator.setString( 0 , "a" );
  memoryAllocator.setString( 1 , "b" );
  memoryAllocator.setString( 5 , "c" );
  memoryAllocator.setString( 1000 , "a b c" );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readText() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readText().toString() , "a b c" );
}
 

@Test
public void T_setString_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new StringObj( "a" ) , 0 );
  column.add( ColumnType.STRING , new StringObj( "b" ) , 1 );
  column.add( ColumnType.STRING , new StringObj( "c" ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeDumpStringColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readText().toString() , "a" );
  reader.setPosition( 1 );
  assertEquals( reader.readText().toString() , "b" );
  reader.setPosition( 5 );
  assertEquals( reader.readText().toString() , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readText() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readText() , null );
}
 

@Test
public void T_setBytes_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 4 );

  memoryAllocator.setBytes( 0 , "a".getBytes() );
  memoryAllocator.setBytes( 1 , "b".getBytes() );
  memoryAllocator.setBytes( 5 , "c".getBytes() );
  memoryAllocator.setBytes( 1000 , "a b c".getBytes() );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readByteArray() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( new String( reader.readByteArray() ) , "a b c" );
}
 

@Test
public void T_setBytes_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.STRING , "boolean" );
  column.add( ColumnType.STRING , new BytesObj( "a".getBytes() ) , 0 );
  column.add( ColumnType.STRING , new BytesObj( "b".getBytes() ) , 1 );
  column.add( ColumnType.STRING , new BytesObj( "c".getBytes() ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBytesColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( new String( reader.readByteArray() ) , "a" );
  reader.setPosition( 1 );
  assertEquals( new String( reader.readByteArray() ) , "b" );
  reader.setPosition( 5 );
  assertEquals( new String( reader.readByteArray() ) , "c" );
  reader.setPosition( 2 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByteArray() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByteArray() , null );
}
 

@Test
public void T_setFloat_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , "target" , allocator , parent , 4 );

  memoryAllocator.setFloat( 0 , (float)0.1 );
  memoryAllocator.setFloat( 1 , (float)0.2 );
  memoryAllocator.setFloat( 5 , (float)0.255 );
  memoryAllocator.setFloat( 1000 , (float)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readFloat().floatValue() , (float)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readFloat().floatValue() , (float)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readFloat().floatValue() , (float)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readFloat() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readFloat().floatValue() , (float)0.1 );
}
 

@Test
public void T_setFloat_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.FLOAT , "boolean" );
  column.add( ColumnType.FLOAT , new FloatObj( (float)100 ) , 0 );
  column.add( ColumnType.FLOAT , new FloatObj( (float)200 ) , 1 );
  column.add( ColumnType.FLOAT , new FloatObj( (float)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeFloatColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readFloat().floatValue() , (float)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readFloat().floatValue() , (float)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readFloat().floatValue() , (float)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readFloat() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readFloat() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readFloat() , null );
}
 

@Test
public void T_setBoolean_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BOOLEAN , "boolean" );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 0 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( false ) , 1 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBooleanColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 2 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readBoolean() , null );
}
 

@Test
public void T_setInteger_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.INTEGER , "boolean" );
  column.add( ColumnType.INTEGER , new IntegerObj( 100 ) , 0 );
  column.add( ColumnType.INTEGER , new IntegerObj( 200 ) , 1 );
  column.add( ColumnType.INTEGER , new IntegerObj( 255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readInteger().intValue() , 100 );
  reader.setPosition( 1 );
  assertEquals( reader.readInteger().intValue() , 200 );
  reader.setPosition( 5 );
  assertEquals( reader.readInteger().intValue() , 255 );
  reader.setPosition( 2 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readInteger() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readInteger() , null );
}
 

@Test
public void T_setShort_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 1001 );

  memoryAllocator.setShort( 0 , (short)100 );
  memoryAllocator.setShort( 1 , (short)200 );
  memoryAllocator.setShort( 5 , (short)255 );
  memoryAllocator.setShort( 1000 , (short)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readShort() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( (short)( reader.readShort() ) , (short)10 );
}
 

@Test
public void T_setShort_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.SHORT , "boolean" );
  column.add( ColumnType.SHORT , new ShortObj( (short)100 ) , 0 );
  column.add( ColumnType.SHORT , new ShortObj( (short)200 ) , 1 );
  column.add( ColumnType.SHORT , new ShortObj( (short)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( (short)( reader.readShort() ) , (short)100 );
  reader.setPosition( 1 );
  assertEquals( (short)( reader.readShort() ) , (short)200 );
  reader.setPosition( 5 );
  assertEquals( (short)( reader.readShort() ) , (short)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readShort() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readShort() , null );
}
 

@Test
public void T_setByte_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 1001 );

  memoryAllocator.setByte( 0 , (byte)100 );
  memoryAllocator.setByte( 1 , (byte)200 );
  memoryAllocator.setByte( 5 , (byte)255 );
  memoryAllocator.setByte( 1000 , (byte)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readByte() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readByte().byteValue() , (byte)10 );
}
 

@Test
public void T_setByte_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BYTE , "boolean" );
  column.add( ColumnType.BYTE , new ByteObj( (byte)100 ) , 0 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)200 ) , 1 );
  column.add( ColumnType.BYTE , new ByteObj( (byte)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new OptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , "target" , allocator , parent , 6 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readByte().byteValue() , (byte)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readByte().byteValue() , (byte)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readByte().byteValue() , (byte)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readByte() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readByte() , null );
}
 

@Test
public void T_setArray_1() throws IOException{
  IColumn column = new ArrayColumn( "array" );
  List<Object> value = new ArrayList<Object>();
  value.add( new StringObj( "a" ) );
  value.add( new StringObj( "b" ) );
  value.add( new StringObj( "c" ) );
  column.add( ColumnType.ARRAY , value , 0 );
  column.add( ColumnType.ARRAY , value , 1 );
  column.add( ColumnType.ARRAY , value , 2 );
  column.add( ColumnType.ARRAY , value , 3 );
  column.add( ColumnType.ARRAY , value , 6 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpArrayColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();

  ListVector listVector = parent.addOrGetList( "target" );
  IMemoryAllocator memoryAllocator = new ArrowArrayMemoryAllocator( allocator , listVector , 7 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  for( int i = 0 ; i < 7 ; i++ ){
    System.out.println( "count:" + listVector.getInnerValueCountAt(i) );
    System.out.println( "obj:" + listVector.getObject(i) );
  }
}
 

public boolean isComparable() {
  switch(type.getTypeID()) {
    case Struct:
    case List:
      return false;
    default:
      return true;
  }
}
 

public boolean isScalar() {
  switch(type.getTypeID()){
  case List:
  case Struct:
  case Union:
    return false;
  default:
    return true;
  }
}
 

public boolean isFixedWidthScalar() {
  switch(type.getTypeID()){
  case List:
  case Struct:
  case Union:
  case Binary:
  case Utf8:
    return false;
  default:
    return true;
  }
}
 

@Test
public void testDataSetSchema() throws Exception {
  Field field1 = new Field("a", true, new Int(32, true), null);
  Field child1 = new Field("c", true, Utf8.INSTANCE, null);
  Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1));
  Schema schema = new Schema(ImmutableList.of(field1, field2));
  FlatBufferBuilder builder = new FlatBufferBuilder();
  schema.getSchema(builder);
  builder.finish(schema.getSchema(builder));
  NamespaceTestUtils.addSource(namespaceService, "s");
  NamespaceTestUtils.addPhysicalDS(namespaceService, "s.foo", builder.sizedByteArray());
  ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(namespaceService.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray());
  Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb));
  assertEquals(schema, returnedSchema);
}
 

@Test
public void T_setBoolean_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.BOOLEAN , "boolean" );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 0 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( false ) , 1 );
  column.add( ColumnType.BOOLEAN , new BooleanObj( true ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpBooleanColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , "target" , allocator , parent , 3 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 1 );
  assertEquals( reader.readBoolean().booleanValue() , false );
  reader.setPosition( 5 );
  assertEquals( reader.readBoolean().booleanValue() , true );
  reader.setPosition( 2 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readBoolean() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readBoolean() , null );
}
 

/**
 * FileReader and Arrow memory allocators are set and initialized.
 */
public DynamicArrowLoader(
    final IRootMemoryAllocator rootMemoryAllocator ,
    final YosegiReader reader ,
    final BufferAllocator allocator ) {
  this.reader = reader;
  this.allocator = allocator;
  this.rootMemoryAllocator = rootMemoryAllocator;
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  rootVector = new StructVector(
      "root" , allocator , new FieldType( true , Struct.INSTANCE , null , null ) , callBack );
}
 

@Test
public void T_setLong_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 1001 );

  memoryAllocator.setLong( 0 , (long)100 );
  memoryAllocator.setLong( 1 , (long)200 );
  memoryAllocator.setLong( 5 , (long)255 );
  memoryAllocator.setLong( 1000 , (long)10 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readLong() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readLong().longValue() , (long)10 );
}
 

@Test
public void T_setLong_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.LONG , "boolean" );
  column.add( ColumnType.LONG , new LongObj( (long)100 ) , 0 );
  column.add( ColumnType.LONG , new LongObj( (long)200 ) , 1 );
  column.add( ColumnType.LONG , new LongObj( (long)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeLongColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readLong().longValue() , (long)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readLong().longValue() , (long)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readLong().longValue() , (long)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readLong() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readLong() , null );
}
 

@Test
public void T_setDouble_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 4 );

  memoryAllocator.setDouble( 0 , (double)0.1 );
  memoryAllocator.setDouble( 1 , (double)0.2 );
  memoryAllocator.setDouble( 5 , (double)0.255 );
  memoryAllocator.setDouble( 1000 , (double)0.1 );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)0,1 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.2 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.255 );
  for( int i = 6 ; i < 1000 ; i++ ){
    reader.setPosition( i );
    assertEquals( reader.readDouble() , null );
  }
  reader.setPosition( 1000 );
  assertEquals( reader.readDouble().doubleValue() , (double)0.1 );
}
 

@Test
public void T_setDouble_2() throws IOException{
  IColumn column = new PrimitiveColumn( ColumnType.DOUBLE , "boolean" );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)100 ) , 0 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)200 ) , 1 );
  column.add( ColumnType.DOUBLE , new DoubleObj( (double)255 ) , 5 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new UnsafeOptimizeDoubleColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();
  IMemoryAllocator memoryAllocator = ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , "target" , allocator , parent , 3 );

  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  StructReader rootReader = parent.getReader();
  FieldReader reader = rootReader.reader( "target" );
  reader.setPosition( 0 );
  assertEquals( reader.readDouble().doubleValue() , (double)100 );
  reader.setPosition( 1 );
  assertEquals( reader.readDouble().doubleValue() , (double)200 );
  reader.setPosition( 5 );
  assertEquals( reader.readDouble().doubleValue() , (double)255 );
  reader.setPosition( 2 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 3 );
  assertEquals( reader.readDouble() , null );
  reader.setPosition( 4 );
  assertEquals( reader.readDouble() , null );
}