Java Code Examples for org.apache.arrow.vector.IntVector#setValueCount()

The following examples show how to use org.apache.arrow.vector.IntVector#setValueCount() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestArrowIntegerConnector.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_convert_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  IntVector vector = new IntVector( "test" , allocator );
  vector.allocateNew();
  vector.setSafe( 0 , 0 );
  vector.setSafe( 1 , 1 );
  vector.setSafe( 2 , 0 );
  vector.setNull( 3 );
  vector.setSafe( 4 , 1 );
  vector.setSafe( 5 , 1 );
  vector.setSafe( 6 , 1 );
  vector.setNull( 7 );
  vector.setValueCount( 8 );

  IColumn column = ArrowColumnFactory.convert( "test" , vector );
  assertEquals( column.getColumnName() , "test" );
  assertEquals( column.size() , 8 );
  assertTrue( ( column.getColumnType() == ColumnType.INTEGER ) );
  assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getInt() , 0  );
  assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getInt() , 1  );
  assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getInt() , 0  );
  assertEquals( column.get(3).getRow() , null  );
  assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getInt() , 1 );
  assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getInt() , 1 );
  assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getInt() , 1 );
  assertEquals( column.get(7).getRow() , null  );
}
Example 2
Source File: VectorizedColumnIterator.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
public void nextBatchDictionaryIds(IntVector vector, NullabilityHolder holder) {
  int rowsReadSoFar = 0;
  while (rowsReadSoFar < batchSize && hasNext()) {
    advance();
    int rowsInThisBatch = vectorizedPageIterator.nextBatchDictionaryIds(vector, batchSize - rowsReadSoFar,
        rowsReadSoFar, holder);
    rowsReadSoFar += rowsInThisBatch;
    this.triplesRead += rowsInThisBatch;
    vector.setValueCount(rowsReadSoFar);
  }
}
Example 3
Source File: TestArrowIntegerConnector.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
@Test
public void T_convert_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  IntVector vector = new IntVector( "test" , allocator );
  vector.allocateNew();
  vector.setSafe( 0 , 0 );  
  vector.setSafe( 1 , 1 );  
  vector.setSafe( 2 , 0 );  
  vector.setNull( 3 );  
  vector.setSafe( 4 , 1 );  
  vector.setSafe( 5 , 1 );  
  vector.setSafe( 6 , 1 );  
  vector.setNull( 7 );  
  vector.setValueCount( 8 );

  IColumn column = ArrowColumnFactory.convert( "test" , vector );
  assertEquals( column.getColumnName() , "test" );
  assertEquals( column.size() , 8 );
  assertTrue( ( column.getColumnType() == ColumnType.INTEGER ) );
  assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getInt() , 0  );
  assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getInt() , 1  );
  assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getInt() , 0  );
  assertEquals( column.get(3).getRow() , null  );
  assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getInt() , 1 );
  assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getInt() , 1 );
  assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getInt() , 1 );
  assertEquals( column.get(7).getRow() , null  );
}
Example 4
Source File: Twister2ArrowFileWriter.java    From twister2 with Apache License 2.0 5 votes vote down vote up 
@Override
public <T extends FieldVector> void generate(T intVector1, int from, int items, int isSet) {
  IntVector intVector = (IntVector) intVector1;
  intVector.setInitialCapacity(items);
  intVector.allocateNew();
  for (int i = 0; i < items; i++) {
    intVector.setSafe(i, isSet, (int) dataList.get(from + i));
  }
  intVector.setValueCount(items);
}
Example 5
Source File: GlobalDictionaryBuilder.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private static VectorContainer buildIntegerGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(32, true), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final IntVector intVector = input.addOrGet(field);
  intVector.allocateNew();
  final SortedSet<Integer> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToInt(i));
    }
  }
  if (existingDict != null) {
    final IntVector existingDictValues = existingDict.getValueAccessorById(IntVector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.get(i));
    }
  }
  final Iterator<Integer> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    intVector.setSafe(recordCount++, iter.next());
  }
  intVector.setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
Example 6
Source File: TestQueryReAttempt.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private IntVector intVector(String name) {
  IntVector vec = new IntVector(name, allocator);
  vec.allocateNew(5);
  vec.set(0, 20);
  vec.set(1, 50);
  vec.set(2, -2000);
  vec.set(3, 327345);
  vec.setNull(4);

  vec.setValueCount(COUNT);
  return vec;
}
Example 7
Source File: TestVectorizedHashAggPartitionSerializable.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private void populateInt(IntVector vector, Integer[] data) {
  vector.allocateNew();
  Random r = new Random();
  for(int i =0; i < data.length; i++){
    Integer val = data[i];
    if(val != null){
      vector.setSafe(i, val);
    } else {
      vector.setSafe(i, 0, r.nextInt());
    }
  }
  vector.setValueCount(data.length);
}
Example 8
Source File: TestIntPivot.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
static void populate(IntVector vector, Integer[] values){
  vector.allocateNew();
  Random r = new Random();
  for(int i =0; i < values.length; i++){
    Integer val = values[i];
    if(val != null){
      vector.setSafe(i, val);
    } else {
      // add noise so we make sure not to read/see noise.
      vector.setSafe(i, 0, r.nextInt());
    }
  }
  vector.setValueCount(values.length);
}
Example 9
Source File: TestPreallocation.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private void populateInt(IntVector vector, Integer[] data) {
  vector.allocateNew();
  Random r = new Random();
  for(int i =0; i < data.length; i++){
    Integer val = data[i];
    if(val != null){
      vector.setSafe(i, val);
    } else {
      vector.setSafe(i, 0, r.nextInt());
    }
  }
  vector.setValueCount(data.length);
}
Example 10
Source File: TestBoundedPivots.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
static Integer[] populate4ByteValues(IntVector vector, int size){
  vector.allocateNew();
  Integer values[] = new Integer[size];
  for(int i =0; i < size; i++){
    if(RAND.nextBoolean()){
      values[i] = RAND.nextInt();
      vector.setSafe(i, values[i]);
    }
  }
  vector.setValueCount(size);
  return values;
}
Example 11
Source File: TestBoundedPivots.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
static Integer[] populate4ByteValuesWithoutNull(IntVector vector, int size){
  vector.allocateNew();
  Integer values[] = new Integer[size];
  for(int i =0; i < size; i++){
    values[i] = RAND.nextInt();
    vector.setSafe(i, values[i]);
  }
  vector.setValueCount(size);
  return values;
}
Example 12
Source File: TestVectorizedHashAggPartitionSpillHandler.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private void populateInt(IntVector vector, Integer[] data) {
  vector.allocateNew();
  Random r = new Random();
  for(int i =0; i < data.length; i++){
    Integer val = data[i];
    if(val != null){
      vector.setSafe(i, val);
    } else {
      vector.setSafe(i, 0, r.nextInt());
    }
  }
  vector.setValueCount(data.length);
}
Example 13
Source File: MockHiveWarehouseConnector.java    From spark-llap with Apache License 2.0 5 votes vote down vote up 
public MockLlapArrowBatchRecordReader(long arrowAllocatorMax) {
  BufferAllocator allocator = RootAllocatorFactory.INSTANCE.getOrCreateRootAllocator(arrowAllocatorMax);
  IntVector vector = new IntVector("a", allocator);
  vector.allocateNewSafe();
  for(int i = 0; i < testVector.length; i++) {
    vector.set(i, testVector[i]);
  }
  vector.setValueCount(testVector.length);
  List<Field> fields = Lists.newArrayList(vector.getField());
  List<FieldVector> vectors = new ArrayList<>();
  vectors.add(vector);
  vectorSchemaRoot = new VectorSchemaRoot(fields, vectors, testVector.length);
}
Example 14
Source File: TestPivotRoundtrip.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
@Test
public void intManyRoundtrip() throws Exception {
  final int count = 1024;
  final int mult = 80;

  ValueVector[] in = new IntVector[mult];
  ValueVector[] out = new IntVector[mult];
  List<FieldVectorPair> pairs = new ArrayList<>();
  try {

    for (int x = 0; x < mult; x++) {
      IntVector inv = new IntVector("in", allocator);
      in[x] = inv;
      inv.allocateNew(count);
      IntVector outv = new IntVector("out", allocator);
      out[x] = outv;
      for (int i = 0; i < count; i++) {
        if (i % 5 == 0) {
          inv.setSafe(i, Integer.MAX_VALUE - i);
        }
      }
      inv.setValueCount(count);
      pairs.add(new FieldVectorPair(inv, outv));
    }

    final PivotDef pivot = PivotBuilder.getBlockDefinition(pairs);
    try (
      final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth());
      final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount());
    ) {
      fbv.ensureAvailableBlocks(count);

      for (int x = 0; x < mult; x++) {
        Pivots.pivot4Bytes(pivot.getFixedPivots().get(x), fbv, count);
      }

      unpivotHelper(pivot, fbv, vbv, in, out, 0, count);
      unpivotHelper(pivot, fbv, vbv, in, out, 0, 100);
      unpivotHelper(pivot, fbv, vbv, in, out, 100, 924);
    }
  } finally {
    AutoCloseables.close(ImmutableList.copyOf(in));
    AutoCloseables.close(ImmutableList.copyOf(out));
  }
}