Java Code Examples for org.apache.arrow.vector.complex.StructVector#addOrGetList()

The following examples show how to use org.apache.arrow.vector.complex.StructVector#addOrGetList() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestArrowArrayMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setArray_1() throws IOException{
  IColumn column = new ArrayColumn( "array" );
  List<Object> value = new ArrayList<Object>();
  value.add( new StringObj( "a" ) );
  value.add( new StringObj( "b" ) );
  value.add( new StringObj( "c" ) );
  column.add( ColumnType.ARRAY , value , 0 );
  column.add( ColumnType.ARRAY , value , 1 );
  column.add( ColumnType.ARRAY , value , 2 );
  column.add( ColumnType.ARRAY , value , 3 );
  column.add( ColumnType.ARRAY , value , 6 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpArrayColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , new CompressResultNode() , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();

  ListVector listVector = parent.addOrGetList( "target" );
  IMemoryAllocator memoryAllocator = new ArrowArrayMemoryAllocator( allocator , listVector , 7 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  for( int i = 0 ; i < 7 ; i++ ){
    System.out.println( "count:" + listVector.getInnerValueCountAt(i) );
    System.out.println( "obj:" + listVector.getObject(i) );
  }
}
Example 2
Source File: ArrowFixedSchemaMemoryAllocatorFactory.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
public static IMemoryAllocator getFromStructVector( final IField schema , final String columnName , final BufferAllocator allocator , final StructVector vector , final int rowCount ) throws IOException{
  switch( schema.getFieldType() ){
    case UNION:
      return NullMemoryAllocator.INSTANCE;
    case ARRAY:
      return new ArrowFixedSchemaArrayMemoryAllocator( (ArrayContainerField)schema , allocator , vector.addOrGetList( columnName ) , rowCount );
    case MAP:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowFixedSchemaMapMemoryAllocator( (MapContainerField)schema , allocator , mapVector , rowCount );
    case STRUCT:
      StructVector structVector = vector.addOrGetStruct( columnName );
      return new ArrowFixedSchemaStructMemoryAllocator( (StructContainerField)schema , allocator , structVector , rowCount );

    case BOOLEAN:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BOOLEAN , columnName ,  allocator , vector , rowCount );
    case BYTE:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTE , columnName ,  allocator , vector , rowCount );
    case SHORT:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.SHORT , columnName ,  allocator , vector , rowCount );
    case INTEGER:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.INTEGER , columnName ,  allocator , vector , rowCount );
    case LONG:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.LONG , columnName ,  allocator , vector , rowCount );
    case FLOAT:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.FLOAT , columnName ,  allocator , vector , rowCount );
    case DOUBLE:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.DOUBLE , columnName ,  allocator , vector , rowCount );
    case STRING:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.STRING , columnName ,  allocator , vector , rowCount );
    case BYTES:
      return ArrowMemoryAllocatorFactory.getFromStructVector( ColumnType.BYTES , columnName ,  allocator , vector , rowCount );

    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example 3
Source File: TestArrowArrayMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_setArray_1() throws IOException{
  IColumn column = new ArrayColumn( "array" );
  List<Object> value = new ArrayList<Object>();
  value.add( new StringObj( "a" ) );
  value.add( new StringObj( "b" ) );
  value.add( new StringObj( "c" ) );
  column.add( ColumnType.ARRAY , value , 0 );
  column.add( ColumnType.ARRAY , value , 1 );
  column.add( ColumnType.ARRAY , value , 2 );
  column.add( ColumnType.ARRAY , value , 3 );
  column.add( ColumnType.ARRAY , value , 6 );

  ColumnBinaryMakerConfig defaultConfig = new ColumnBinaryMakerConfig();
  ColumnBinaryMakerCustomConfigNode configNode = new ColumnBinaryMakerCustomConfigNode( "root" , defaultConfig );

  IColumnBinaryMaker maker = new DumpArrayColumnBinaryMaker();
  ColumnBinary columnBinary = maker.toBinary( defaultConfig , null , column );

  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  SchemaChangeCallBack callBack = new SchemaChangeCallBack();
  StructVector parent = new StructVector("root", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
  parent.allocateNew();

  ListVector listVector = parent.addOrGetList( "target" );
  IMemoryAllocator memoryAllocator = new ArrowArrayMemoryAllocator( allocator , listVector , 7 );
  maker.loadInMemoryStorage( columnBinary , memoryAllocator );

  for( int i = 0 ; i < 7 ; i++ ){
    System.out.println( "count:" + listVector.getInnerValueCountAt(i) );
    System.out.println( "obj:" + listVector.getObject(i) );
  }
}
Example 4
Source File: ArrowFixedSchemaMemoryAllocatorFactory.java    From yosegi with Apache License 2.0 4 votes vote down vote up 
/**
 * Function to convert to IMemoryAllocator for Struct vector.
 */
public static IMemoryAllocator getFromStructVector(
      final IField schema ,
      final String columnName ,
      final BufferAllocator allocator ,
      final StructVector vector ,
      final int rowCount ) throws IOException {
  switch ( schema.getFieldType() ) {
    case UNION:
      return NullMemoryAllocator.INSTANCE;
    case ARRAY:
      return new ArrowFixedSchemaArrayMemoryAllocator(
          (ArrayContainerField)schema ,
          allocator ,
          vector.addOrGetList( columnName ) ,
          rowCount );
    case MAP:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowFixedSchemaMapMemoryAllocator(
          (MapContainerField)schema , allocator , mapVector , rowCount );
    case STRUCT:
      StructVector structVector = vector.addOrGetStruct( columnName );
      return new ArrowFixedSchemaStructMemoryAllocator(
          (StructContainerField)schema , allocator , structVector , rowCount );

    case BOOLEAN:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.BOOLEAN , columnName ,  allocator , vector , rowCount );
    case BYTE:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.BYTE , columnName ,  allocator , vector , rowCount );
    case SHORT:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.SHORT , columnName ,  allocator , vector , rowCount );
    case INTEGER:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.INTEGER , columnName ,  allocator , vector , rowCount );
    case LONG:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.LONG , columnName ,  allocator , vector , rowCount );
    case FLOAT:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.FLOAT , columnName ,  allocator , vector , rowCount );
    case DOUBLE:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.DOUBLE , columnName ,  allocator , vector , rowCount );
    case STRING:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.STRING , columnName ,  allocator , vector , rowCount );
    case BYTES:
      return ArrowMemoryAllocatorFactory.getFromStructVector(
          ColumnType.BYTES , columnName ,  allocator , vector , rowCount );

    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example 5
Source File: ArrowMemoryAllocatorFactory.java    From yosegi with Apache License 2.0 4 votes vote down vote up 
/**
 * Set the vector of Struct and initialize it.
 */
public static IMemoryAllocator getFromStructVector(
    final ColumnType columnType ,
    final String columnName ,
    final BufferAllocator allocator ,
    final StructVector vector ,
    final int rowCount ) {
  switch ( columnType ) {
    case UNION:
      UnionVector unionVector = vector.addOrGetUnion( columnName );
      return new ArrowUnionMemoryAllocator( allocator , unionVector , rowCount );
    case ARRAY:
      return new ArrowArrayMemoryAllocator(
          allocator , vector.addOrGetList( columnName ) , rowCount );
    case SPREAD:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowMapMemoryAllocator( allocator , mapVector , rowCount );

    case BOOLEAN:
      BitVector bitVector =  vector.addOrGet(
          columnName ,
          new FieldType( true , ArrowType.Bool.INSTANCE , null , null ) ,
          BitVector.class );
      return new ArrowBooleanMemoryAllocator( bitVector , rowCount );
    case BYTE:
      TinyIntVector byteVector =  vector.addOrGet(
          columnName ,
          new FieldType( true , new ArrowType.Int( 8 , true ) , null , null ) ,
          TinyIntVector.class );
      return new ArrowByteMemoryAllocator( byteVector , rowCount );
    case SHORT:
      SmallIntVector shortVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.Int( 16 , true ) ,
            null ,
            null ) ,
          SmallIntVector.class );
      return new ArrowShortMemoryAllocator( shortVector , rowCount );
    case INTEGER:
      IntVector integerVector =  vector.addOrGet(
          columnName ,
          new FieldType( true , new ArrowType.Int( 32 , true ) , null , null ) ,
          IntVector.class );
      return new ArrowIntegerMemoryAllocator( integerVector , rowCount );
    case LONG:
      BigIntVector longVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.Int( 64 , true ) ,
            null ,
            null ) ,
            BigIntVector.class );
      return new ArrowLongMemoryAllocator( longVector , rowCount );
    case FLOAT:
      Float4Vector floatVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.FloatingPoint( FloatingPointPrecision.SINGLE ) ,
            null ,
            null ) ,
          Float4Vector.class );
      return new ArrowFloatMemoryAllocator( floatVector , rowCount );
    case DOUBLE:
      Float8Vector doubleVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.FloatingPoint( FloatingPointPrecision.DOUBLE ) ,
            null ,
            null ) ,
          Float8Vector.class );
      return new ArrowDoubleMemoryAllocator( doubleVector , rowCount );
    case STRING:
      VarCharVector charVector = vector.addOrGet(
          columnName ,
          new FieldType( true , ArrowType.Utf8.INSTANCE , null , null ) ,
          VarCharVector.class );
      return new ArrowStringMemoryAllocator( charVector , rowCount );
    case BYTES:
      VarBinaryVector binaryVector = vector.addOrGet(
          columnName ,
          new FieldType( true , ArrowType.Binary.INSTANCE , null , null ) ,
          VarBinaryVector.class );
      return new ArrowBytesMemoryAllocator( binaryVector , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example 6
Source File: ArrowMemoryAllocatorFactory.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up 
public static IMemoryAllocator getFromStructVector( final ColumnType columnType , final String columnName , final BufferAllocator allocator , final StructVector vector , final int rowCount ){
  switch( columnType ){
    case UNION:
      UnionVector unionVector = vector.addOrGetUnion( columnName );
      return new ArrowUnionMemoryAllocator( allocator , unionVector , rowCount );
    case ARRAY:
      return new ArrowArrayMemoryAllocator( allocator , vector.addOrGetList( columnName ) , rowCount );
    case SPREAD:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowMapMemoryAllocator( allocator , mapVector , rowCount );

    case BOOLEAN:
      BitVector bitVector =  vector.addOrGet( columnName , new FieldType( true , ArrowType.Bool.INSTANCE , null , null ) , BitVector.class );
      return new ArrowBooleanMemoryAllocator( bitVector , rowCount );
    case BYTE:
      TinyIntVector byteVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 8 , true ) , null , null ) , TinyIntVector.class );
      return new ArrowByteMemoryAllocator( byteVector , rowCount );
    case SHORT:
      SmallIntVector shortVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 16 , true ) , null , null ) , SmallIntVector.class );
      return new ArrowShortMemoryAllocator( shortVector , rowCount );
    case INTEGER:
      IntVector integerVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 32 , true ) , null , null ) , IntVector.class );
      return new ArrowIntegerMemoryAllocator( integerVector , rowCount );
    case LONG:
      BigIntVector longVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 64 , true ) , null , null ) , BigIntVector.class );
      return new ArrowLongMemoryAllocator( longVector , rowCount );
    case FLOAT:
      Float4Vector floatVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.FloatingPoint( FloatingPointPrecision.SINGLE ) , null , null ) , Float4Vector.class );
      return new ArrowFloatMemoryAllocator( floatVector , rowCount );
    case DOUBLE:
      Float8Vector doubleVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.FloatingPoint( FloatingPointPrecision.DOUBLE ) , null , null ) , Float8Vector.class );
      return new ArrowDoubleMemoryAllocator( doubleVector , rowCount );
    case STRING:
      VarCharVector charVector =  vector.addOrGet( columnName , new FieldType( true , ArrowType.Utf8.INSTANCE , null , null ) , VarCharVector.class );
      return new ArrowStringMemoryAllocator( charVector , rowCount );
    case BYTES:
      VarBinaryVector binaryVector =  vector.addOrGet( columnName , new FieldType( true , ArrowType.Binary.INSTANCE , null , null ) , VarBinaryVector.class );
      return new ArrowBytesMemoryAllocator( binaryVector , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}