org.apache.hadoop.hive.ql.exec.vector.LongColumnVector Java Examples

@Override
public void process(WayContainer container) {
    DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
    DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];
    ListColumnVector nds = (ListColumnVector) batch.cols[5];

    checkLimit();
    addCommonProperties(container);

    lat.isNull[row] = true;
    lon.isNull[row] = true;
    lat.set(row, (HiveDecimal) null);
    lon.set(row, (HiveDecimal) null);

    Way way = container.getEntity();

    nds.lengths[row] = way.getWayNodes().size();
    nds.childCount += nds.lengths[row];
    nds.child.ensureSize(nds.childCount, nds.offsets[row] != 0);

    for (int j = 0; j < way.getWayNodes().size(); j++) {
        StructColumnVector ndsStruct = (StructColumnVector) nds.child;

        ((LongColumnVector) ndsStruct.fields[0]).vector[(int) nds.offsets[row] + j] = way.getWayNodes().get(j).getNodeId();
    }
}
 

private static List<Record> getResults(Reader reader) throws IOException {
	List<Record> results = new ArrayList<>();

	RecordReader recordReader = reader.rows();
	VectorizedRowBatch batch = reader.getSchema().createRowBatch();

	while (recordReader.nextBatch(batch)) {
		BytesColumnVector stringVector = (BytesColumnVector)  batch.cols[0];
		LongColumnVector intVector = (LongColumnVector) batch.cols[1];
		for (int r = 0; r < batch.size; r++) {
			String name = new String(stringVector.vector[r], stringVector.start[r], stringVector.length[r]);
			int age = (int) intVector.vector[r];

			results.add(new Record(name, age));
		}
		recordReader.close();
	}

	return results;
}
 

private static <T> void readNonNullLongColumn(Object[] vals, int fieldIdx, LongColumnVector vector,
												int childCount, LongFunction<T> reader) {

	if (vector.isRepeating) { // fill complete column with first value
		T repeatingValue = reader.apply(vector.vector[0]);
		fillColumnWithRepeatingValue(vals, fieldIdx, repeatingValue, childCount);
	} else {
		if (fieldIdx == -1) { // set as an object
			for (int i = 0; i < childCount; i++) {
				vals[i] = reader.apply(vector.vector[i]);
			}
		} else { // set as a field of Row
			Row[] rows = (Row[]) vals;
			for (int i = 0; i < childCount; i++) {
				rows[i].setField(fieldIdx, reader.apply(vector.vector[i]));
			}
		}
	}
}
 

private static <T> void readNonNullLongColumn(Object[] vals, int fieldIdx, LongColumnVector vector,
												int childCount, LongFunction<T> reader) {

	if (vector.isRepeating) { // fill complete column with first value
		T repeatingValue = reader.apply(vector.vector[0]);
		fillColumnWithRepeatingValue(vals, fieldIdx, repeatingValue, childCount);
	} else {
		if (fieldIdx == -1) { // set as an object
			for (int i = 0; i < childCount; i++) {
				vals[i] = reader.apply(vector.vector[i]);
			}
		} else { // set as a field of Row
			Row[] rows = (Row[]) vals;
			for (int i = 0; i < childCount; i++) {
				rows[i].setField(fieldIdx, reader.apply(vector.vector[i]));
			}
		}
	}
}
 

protected StringDictionaryTreeReader(int columnId, InStream present, InStream data,
                                     InStream length, InStream dictionary, OrcProto.ColumnEncoding encoding)
    throws IOException {
  super(columnId, present);
  scratchlcv = new LongColumnVector();
  if (data != null && encoding != null) {
    this.reader = createIntegerReader(encoding.getKind(), data, false, false);
  }

  if (dictionary != null && encoding != null) {
    readDictionaryStream(dictionary);
  }

  if (length != null && encoding != null) {
    readDictionaryLengthStream(length, encoding);
  }
}
 

private static <T> void readNonNullLongColumn(Object[] vals, int fieldIdx, LongColumnVector vector,
												int childCount, LongFunction<T> reader) {

	if (vector.isRepeating) { // fill complete column with first value
		T repeatingValue = reader.apply(vector.vector[0]);
		fillColumnWithRepeatingValue(vals, fieldIdx, repeatingValue, childCount);
	} else {
		if (fieldIdx == -1) { // set as an object
			for (int i = 0; i < childCount; i++) {
				vals[i] = reader.apply(vector.vector[i]);
			}
		} else { // set as a field of Row
			Row[] rows = (Row[]) vals;
			for (int i = 0; i < childCount; i++) {
				rows[i].setField(fieldIdx, reader.apply(vector.vector[i]));
			}
		}
	}
}
 

protected BinaryTreeReader(int columnId, InStream present, InStream data, InStream length,
                           OrcProto.ColumnEncoding encoding) throws IOException {
  super(columnId, present);
  scratchlcv = new LongColumnVector();
  this.stream = data;
  if (length != null && encoding != null) {
    checkEncoding(encoding);
    this.lengths = createIntegerReader(encoding.getKind(), length, false, false);
  }
}
 

protected StringDirectTreeReader(int columnId, InStream present, InStream data,
                                 InStream length, OrcProto.ColumnEncoding.Kind encoding) throws IOException {
  super(columnId, present);
  this.scratchlcv = new LongColumnVector();
  this.stream = data;
  if (length != null && encoding != null) {
    this.lengths = createIntegerReader(encoding, length, false, false);
  }
}
 

public File createORCFile(int[] values) throws IOException {
  File file = File.createTempFile("zeppelin-flink-input", ".orc");
  file.delete();
  Path path = new Path(file.getAbsolutePath());
  Configuration conf = new Configuration();
  conf.set("orc.compress", "snappy");
  TypeDescription schema = TypeDescription.fromString("struct<msg:int>");
  Writer writer = OrcFile.createWriter(path,
          OrcFile.writerOptions(conf)
                  .setSchema(schema));
  VectorizedRowBatch batch = schema.createRowBatch();
  LongColumnVector x = (LongColumnVector) batch.cols[0];
  for (int i = 0; i < values.length; ++i) {
    int row = batch.size++;
    x.vector[row] = values[i];
    // If the batch is full, write it out and start over.
    if (batch.size == batch.getMaxSize()) {
      writer.addRowBatch(batch);
      batch.reset();
    }
  }
  if (batch.size != 0) {
    writer.addRowBatch(batch);
    batch.reset();
  }
  writer.close();
  return file;
}
 

private static <T> void readLongColumn(Object[] vals, int fieldIdx, LongColumnVector vector,
										int childCount, LongFunction<T> reader) {

	if (vector.isRepeating) { // fill complete column with first value
		if (vector.isNull[0]) {
			// fill vals with null values
			fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
		} else {
			// read repeating non-null value by forwarding call.
			readNonNullLongColumn(vals, fieldIdx, vector, childCount, reader);
		}
	} else {
		boolean[] isNullVector = vector.isNull;
		if (fieldIdx == -1) { // set as an object
			for (int i = 0; i < childCount; i++) {
				if (isNullVector[i]) {
					vals[i] = null;
				} else {
					vals[i] = reader.apply(vector.vector[i]);
				}
			}
		} else { // set as a field of Row
			Row[] rows = (Row[]) vals;
			for (int i = 0; i < childCount; i++) {
				if (isNullVector[i]) {
					rows[i].setField(fieldIdx, null);
				} else {
					rows[i].setField(fieldIdx, reader.apply(vector.vector[i]));
				}
			}
		}
	}
}
 

@Override
public void process(RelationContainer container) {
    DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
    DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];
    ListColumnVector members = (ListColumnVector) batch.cols[6];

    checkLimit();
    addCommonProperties(container);

    lat.isNull[row] = true;
    lon.isNull[row] = true;
    lat.set(row, (HiveDecimal) null);
    lon.set(row, (HiveDecimal) null);

    Relation relation = container.getEntity();

    members.lengths[row] = relation.getMembers().size();
    members.childCount += members.lengths[row];
    members.child.ensureSize(members.childCount, members.offsets[row] != 0);

    for (int j = 0; j < relation.getMembers().size(); j++) {
        StructColumnVector membersStruct = (StructColumnVector) members.child;

        ((BytesColumnVector) membersStruct.fields[0]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberType().toString().toLowerCase().getBytes());
        ((LongColumnVector) membersStruct.fields[1]).vector[(int) members.offsets[row] + j] = relation.getMembers().get(j).getMemberId();
        ((BytesColumnVector) membersStruct.fields[2]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberRole().getBytes());
    }
}
 

private static void readLongColumnAsDate(Object[] vals, int fieldIdx, LongColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			if (vector.isNull[0]) {
				// fill vals with null values
				fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
			} else {
				// read repeating non-null value by forwarding call
				readNonNullLongColumnAsDate(vals, fieldIdx, vector, childCount);
			}
		} else {
			boolean[] isNullVector = vector.isNull;
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						vals[i] = null;
					} else {
						vals[i] = readDate(vector.vector[i]);
					}
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						rows[i].setField(fieldIdx, null);
					} else {
						rows[i].setField(fieldIdx, readDate(vector.vector[i]));
					}
				}
			}
		}
	}
 

private ColumnVector[] createTransactionalVectors(ColumnVector[] dataVectors) {
  ColumnVector[] transVectors = new ColumnVector[6];

  transVectors[0] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[1] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[2] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[3] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[4] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);

  transVectors[5] = new StructColumnVector(dataVectors.length, dataVectors);

  return transVectors;
}
 

private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case DATE:
      return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case TIMESTAMP:
      return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case FLOAT:
    case DOUBLE:
      return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case DECIMAL:
      DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
      return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
        tInfo.precision(), tInfo.scale()
      );
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
        .build(logger);
    }
}
 

@Test
public void testListCopier() {
  LongColumnVector input1 = new LongColumnVector(HIVE_BATCH_SIZE);
  ListColumnVector input = new ListColumnVector(HIVE_BATCH_SIZE, input1);
  input.init();
  getHiveBatch(input, input1);

  HiveORCCopiers.ListCopier listCopier = new HiveORCCopiers.ListCopier((MultiValuedColumnVector)input);
  long childcountInFirstHalf = listCopier.countChildren(input.noNulls,
    input.lengths, 0, 512);
  long childcountInSecondHalf = listCopier.countChildren(input.noNulls,
    input.lengths, 512, 512);
  assertEquals(1024, childcountInFirstHalf);
  assertEquals(0, childcountInSecondHalf);
}
 

private void getHiveBatch(ListColumnVector input, LongColumnVector child) {
  input.noNulls = false;

  input.childCount = 800;

  for(int i=0; i<HIVE_BATCH_SIZE; ++i) {
    child.vector[i] = 10 * i;
    input.offsets[i] = i;
    input.lengths[i] = 2;
  }
  for(int i=512; i<HIVE_BATCH_SIZE; ++i) {
    input.isNull[i] = true;
  }
}
 

private static LongColumnVector createLongVector(int batchSize, Object value) {
	LongColumnVector lcv = new LongColumnVector(batchSize);
	if (value == null) {
		lcv.noNulls = false;
		lcv.isNull[0] = true;
		lcv.isRepeating = true;
	} else {
		lcv.fill(((Number) value).longValue());
		lcv.isNull[0] = false;
	}
	return lcv;
}
 

@Override
public void vectorize(Record element, VectorizedRowBatch batch) throws IOException {
	BytesColumnVector stringVector = (BytesColumnVector) batch.cols[0];
	LongColumnVector intColVector = (LongColumnVector) batch.cols[1];

	int row = batch.size++;

	stringVector.setVal(row, element.getName().getBytes(StandardCharsets.UTF_8));
	intColVector.vector[row] = element.getAge();

	this.addUserMetadata(OrcBulkWriterTestUtil.USER_METADATA_KEY, OrcBulkWriterTestUtil.USER_METADATA_VALUE);
}
 

private static <T> void readLongColumn(Object[] vals, int fieldIdx, LongColumnVector vector,
										int childCount, LongFunction<T> reader) {

	if (vector.isRepeating) { // fill complete column with first value
		if (vector.isNull[0]) {
			// fill vals with null values
			fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
		} else {
			// read repeating non-null value by forwarding call.
			readNonNullLongColumn(vals, fieldIdx, vector, childCount, reader);
		}
	} else {
		boolean[] isNullVector = vector.isNull;
		if (fieldIdx == -1) { // set as an object
			for (int i = 0; i < childCount; i++) {
				if (isNullVector[i]) {
					vals[i] = null;
				} else {
					vals[i] = reader.apply(vector.vector[i]);
				}
			}
		} else { // set as a field of Row
			Row[] rows = (Row[]) vals;
			for (int i = 0; i < childCount; i++) {
				if (isNullVector[i]) {
					rows[i].setField(fieldIdx, null);
				} else {
					rows[i].setField(fieldIdx, reader.apply(vector.vector[i]));
				}
			}
		}
	}
}
 

public void convert(JsonElement value, ColumnVector vect, int row) {
    if (value == null || value.isJsonNull()) {
        vect.noNulls = false;
        vect.isNull[row] = true;
    } else {
        LongColumnVector vector = (LongColumnVector) vect;
        vector.vector[row] = value.getAsBoolean() ? 1 : 0;
    }
}
 

public void convert(JsonElement value, ColumnVector vect, int row) {
    if (value == null || value.isJsonNull()) {
        vect.noNulls = false;
        vect.isNull[row] = true;
    } else {
        LongColumnVector vector = (LongColumnVector) vect;
        vector.vector[row] = value.getAsLong();
    }
}
 

private static void readLongColumnAsDate(Object[] vals, int fieldIdx, LongColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			if (vector.isNull[0]) {
				// fill vals with null values
				fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
			} else {
				// read repeating non-null value by forwarding call
				readNonNullLongColumnAsDate(vals, fieldIdx, vector, childCount);
			}
		} else {
			boolean[] isNullVector = vector.isNull;
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						vals[i] = null;
					} else {
						vals[i] = readDate(vector.vector[i]);
					}
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						rows[i].setField(fieldIdx, null);
					} else {
						rows[i].setField(fieldIdx, readDate(vector.vector[i]));
					}
				}
			}
		}
	}
 

private ColumnVector[] createTransactionalVectors(ColumnVector[] dataVectors) {
  ColumnVector[] transVectors = new ColumnVector[6];

  transVectors[0] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[1] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[2] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[3] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
  transVectors[4] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);

  transVectors[5] = new StructColumnVector(dataVectors.length, dataVectors);

  return transVectors;
}
 

private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case DATE:
      return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case TIMESTAMP:
      return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case FLOAT:
    case DOUBLE:
      return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case DECIMAL:
      DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
      return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
        tInfo.precision(), tInfo.scale()
      );
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
        .build(logger);
    }
}
 

private static <T> void readLongColumn(Object[] vals, int fieldIdx, LongColumnVector vector,
										int childCount, LongFunction<T> reader) {

	if (vector.isRepeating) { // fill complete column with first value
		if (vector.isNull[0]) {
			// fill vals with null values
			fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
		} else {
			// read repeating non-null value by forwarding call.
			readNonNullLongColumn(vals, fieldIdx, vector, childCount, reader);
		}
	} else {
		boolean[] isNullVector = vector.isNull;
		if (fieldIdx == -1) { // set as an object
			for (int i = 0; i < childCount; i++) {
				if (isNullVector[i]) {
					vals[i] = null;
				} else {
					vals[i] = reader.apply(vector.vector[i]);
				}
			}
		} else { // set as a field of Row
			Row[] rows = (Row[]) vals;
			for (int i = 0; i < childCount; i++) {
				if (isNullVector[i]) {
					rows[i].setField(fieldIdx, null);
				} else {
					rows[i].setField(fieldIdx, reader.apply(vector.vector[i]));
				}
			}
		}
	}
}
 

@Override
public void set( final PrimitiveObject[] primitiveObjectArray , final LongColumnVector columnVector , final int index ) throws IOException{
  try{
    long longNumber = (long)( primitiveObjectArray[index].getInt() );
    columnVector.vector[index] = longNumber;
  }catch( NumberFormatException | NullPointerException e ){
    VectorizedBatchUtil.setNullColIsNullValue( columnVector , index );
  }
}
 

@Override
public void set( final PrimitiveObject[] primitiveObjectArray , final LongColumnVector columnVector , final int index ) throws IOException{
  try{
    long longNumber = (long)( primitiveObjectArray[index].getShort() );
    columnVector.vector[index] = longNumber;
  }catch( NumberFormatException | NullPointerException e ){
    VectorizedBatchUtil.setNullColIsNullValue( columnVector , index );
  }
}
 

@Override
public void setColumnVector( final ColumnVector vector , final IExpressionIndex indexList , final int start , final int length ) throws IOException{
  LongColumnVector columnVector = (LongColumnVector)vector;
  PrimitiveObject[] primitiveObjectArray = column.getPrimitiveObjectArray( indexList , start , length );
  for( int i = 0 ; i < length ; i++ ){
    if( primitiveObjectArray[i] == null ){
      VectorizedBatchUtil.setNullColIsNullValue( columnVector , i );
    }
    else{
      setter.set( primitiveObjectArray , columnVector , i );
    }
  }
}
 

@Override
public void set( final PrimitiveObject[] primitiveObjectArray , final LongColumnVector columnVector , final int index ) throws IOException{
  try{
    long longNumber = (long)( primitiveObjectArray[index].getByte() );
    columnVector.vector[index] = longNumber;
  }catch( NumberFormatException | NullPointerException e ){
    VectorizedBatchUtil.setNullColIsNullValue( columnVector , index );
  }
}
 

@Override
public void set( final PrimitiveObject[] primitiveObjectArray , final LongColumnVector columnVector , final int index ) throws IOException{
  try{
    columnVector.vector[index] = primitiveObjectArray[index].getLong();
  }catch( NumberFormatException | NullPointerException e ){
    VectorizedBatchUtil.setNullColIsNullValue( columnVector , index );
  }
}