org.apache.arrow.vector.TimeStampMilliVector Java Examples

private static Pair<TimeStampMilliVector, ResultVerifier> testTimeStampVector(final int startIndexInCurrentOutput, final int startIndexInJob) {
  TimeStampMilliVector colTimeStampV = new TimeStampMilliVector("colTimeStamp", allocator);
  colTimeStampV.allocateNew(5);
  colTimeStampV.set(0, 23423234);
  colTimeStampV.set(1, -234223);
  colTimeStampV.setNull(2);
  colTimeStampV.set(3, 384928359237L);
  colTimeStampV.set(4, 234289342983294234L);

  ResultVerifier verifier = new ResultVerifier() {
    @Override
    public void verify(DataPOJO output) {
      int index = startIndexInCurrentOutput;
      assertEquals("1970-01-01 06:30:23.234", output.extractValue("colTimeStamp", index++));
      assertEquals("1969-12-31 23:56:05.777", output.extractValue("colTimeStamp", index++));
      assertNull(output.extractValue("colTimeStamp", index++));
      assertEquals("1982-03-14 04:32:39.237", output.extractValue("colTimeStamp", index++));
      assertEquals("7426303-09-23 10:54:54.234", output.extractValue("colTimeStamp", index++));
    }
  };

  return Pair.of(colTimeStampV, verifier);
}
 

protected ColumnReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                       ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v, SchemaElement schemaElement) throws ExecutionSetupException {
  this.parentReader = parentReader;
  this.columnDescriptor = descriptor;
  this.columnChunkMetaData = columnChunkMetaData;
  this.isFixedLength = fixedLength;
  this.schemaElement = schemaElement;
  this.valueVec =  v;
  this.pageReader = (parentReader.getSingleStream() != null)?
    new DeprecatedSingleStreamPageReader(this, parentReader.getSingleStream(), parentReader.getFsPath(), columnChunkMetaData) :
    new PageReader(this, parentReader.getFileSystem(), parentReader.getFsPath(), columnChunkMetaData);

  if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
    if (columnDescriptor.getType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
      dataTypeLengthInBits = columnDescriptor.getTypeLength() * 8;
    } else if (columnDescriptor.getType() == PrimitiveTypeName.INT96
      && valueVec instanceof TimeStampMilliVector) {
      // if int 96 column is being read as a Timestamp, this truncates the time format used by Impala
      // dataTypeLengthInBits is only ever used when computing offsets into the destination vector, so it
      // needs to be set to the bit width of the resulting Arrow type, usually this matches the input length
      dataTypeLengthInBits = 64;
    } else {
      dataTypeLengthInBits = DeprecatedParquetVectorizedReader.getTypeLengthInBits(columnDescriptor.getType());
    }
  }
}
 

@Test
public void testDates() {
    Date now = new Date();
    BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE);
    TimeStampMilliVector timeStampMilliVector = ArrowConverter.vectorFor(bufferAllocator, "col1", new Date[]{now});
    assertEquals(now.getTime(),timeStampMilliVector.get(0));
}
 

@Test
public void testDates() {
    Date now = new Date();
    BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE);
    TimeStampMilliVector timeStampMilliVector = ArrowConverter.vectorFor(bufferAllocator, "col1", new Date[]{now});
    assertEquals(now.getTime(),timeStampMilliVector.get(0));
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final TimestampWritableV2 value = ((TimestampObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  long seconds = value.getSeconds();
  long nanos = value.getNanos();
  long millis = seconds * 1000 + nanos/1000/1000;
  ((TimeStampMilliVector) outputVV).setSafe(outputIndex, millis);
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final TimestampWritable value = ((TimestampObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  long seconds = value.getSeconds();
  long nanos = value.getNanos();
  long millis = seconds * 1000 + nanos/1000/1000;
  ((TimeStampMilliVector) outputVV).setSafe(outputIndex, millis);
}
 

@Before
public void setUp() {
  valueVector = new TimeStampMilliVector("t", new RootAllocator());
  valueVector.allocateNew(3);
  valueVector.set(0, NON_NULL_VALUE);
  valueVector.set(1, DST_VALUE);
  valueVector.setNull(2);

  accessor = new TimeStampMilliAccessor(valueVector, UTC_CALENDAR.getTimeZone());
}
 

@Override
public void set(ValueVector v, int index) {
  if(obj != null){
    ((TimeStampMilliVector) v).setSafe(index, com.dremio.common.util.DateTimes.toMillis(obj));
  }
}
 

TimeStampMilliCopier(TimestampColumnVector inputVector, TimeStampMilliVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

TimeStampMilliCopier(TimestampColumnVector inputVector, TimeStampMilliVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

public static SqlAccessor getSqlAccessor(ValueVector vector, TimeZone defaultTz) {
  final MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(vector.getField().getType());
  switch(type){
  case UNION:
    return new UnionSqlAccessor((UnionVector) vector);
  case TINYINT:
    return new TinyIntAccessor((TinyIntVector) vector);
  case UINT1:
    return new UInt1Accessor((UInt1Vector) vector);
  case UINT2:
    return new UInt2Accessor((UInt2Vector) vector);
  case SMALLINT:
    return new SmallIntAccessor((SmallIntVector) vector);
  case INT:
    return new IntAccessor((IntVector) vector);
  case UINT4:
    return new UInt4Accessor((UInt4Vector) vector);
  case FLOAT4:
    return new Float4Accessor((Float4Vector) vector);
  case INTERVALYEAR:
    return new IntervalYearAccessor((IntervalYearVector) vector);
  case TIMEMILLI:
    return new TimeMilliAccessor((TimeMilliVector) vector, defaultTz);
  case BIGINT:
    return new BigIntAccessor((BigIntVector) vector);
  case UINT8:
    return new UInt8Accessor((UInt8Vector) vector);
  case FLOAT8:
    return new Float8Accessor((Float8Vector) vector);
  case DATEMILLI:
    return new DateMilliAccessor((DateMilliVector) vector, defaultTz);
  case TIMESTAMPMILLI:
    return new TimeStampMilliAccessor((TimeStampMilliVector) vector, defaultTz);
  case INTERVALDAY:
    return new IntervalDayAccessor((IntervalDayVector) vector);
  case DECIMAL:
    return new DecimalAccessor((DecimalVector) vector);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryAccessor((FixedSizeBinaryVector) vector);
  case VARBINARY:
    return new VarBinaryAccessor((VarBinaryVector) vector);
  case VARCHAR:
    return new VarCharAccessor((VarCharVector) vector);
  case BIT:
    return new BitAccessor((BitVector) vector);
  case STRUCT:
  case LIST:
    return new GenericAccessor(vector);
  }
  throw new UnsupportedOperationException(String.format("Unable to find sql accessor for minor type [%s]", type));
}
 

public TimeStampMilliAccessor(TimeStampMilliVector vector, TimeZone defaultTZ) {
  this.ac = vector;
  this.defaultTimeZone = Preconditions.checkNotNull(defaultTZ, "Null TimeZone supplied.");
}
 

public FlightArrowColumnVector(ValueVector vector) {
  super(FlightArrowUtils.fromArrowField(vector.getField()));

  if (vector instanceof BitVector) {
    accessor = new BooleanAccessor((BitVector) vector);
  } else if (vector instanceof TinyIntVector) {
    accessor = new ByteAccessor((TinyIntVector) vector);
  } else if (vector instanceof SmallIntVector) {
    accessor = new ShortAccessor((SmallIntVector) vector);
  } else if (vector instanceof IntVector) {
    accessor = new IntAccessor((IntVector) vector);
  } else if (vector instanceof BigIntVector) {
    accessor = new LongAccessor((BigIntVector) vector);
  } else if (vector instanceof Float4Vector) {
    accessor = new FloatAccessor((Float4Vector) vector);
  } else if (vector instanceof Float8Vector) {
    accessor = new DoubleAccessor((Float8Vector) vector);
  } else if (vector instanceof DecimalVector) {
    accessor = new DecimalAccessor((DecimalVector) vector);
  } else if (vector instanceof VarCharVector) {
    accessor = new StringAccessor((VarCharVector) vector);
  } else if (vector instanceof VarBinaryVector) {
    accessor = new BinaryAccessor((VarBinaryVector) vector);
  } else if (vector instanceof DateDayVector) {
    accessor = new DateAccessor((DateDayVector) vector);
  } else if (vector instanceof DateMilliVector) {
    accessor = new DateMilliAccessor((DateMilliVector) vector);
  } else if (vector instanceof TimeStampMicroTZVector) {
    accessor = new TimestampAccessor((TimeStampMicroTZVector) vector);
  } else if (vector instanceof TimeStampMilliVector) {
    accessor = new TimestampMilliAccessor((TimeStampMilliVector) vector);
  } else if (vector instanceof ListVector) {
    ListVector listVector = (ListVector) vector;
    accessor = new ArrayAccessor(listVector);
  } else if (vector instanceof StructVector) {
    StructVector structVector = (StructVector) vector;
    accessor = new StructAccessor(structVector);

    childColumns = new FlightArrowColumnVector[structVector.size()];
    for (int i = 0; i < childColumns.length; ++i) {
      childColumns[i] = new FlightArrowColumnVector(structVector.getVectorById(i));
    }
  } else {
    System.out.println(vector);
    throw new UnsupportedOperationException();
  }
}
 

public static NullableColumnReader<?> getNullableColumnReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize,
                                                              ColumnDescriptor columnDescriptor,
                                                              ColumnChunkMetaData columnChunkMetaData,
                                                              boolean fixedLength,
                                                              ValueVector valueVec,
                                                              SchemaElement schemaElement) throws ExecutionSetupException {
  ConvertedType convertedType = schemaElement.getConverted_type();

  if (! columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
    if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.INT96) {
       // TODO: check convertedType once parquet support TIMESTAMP_NANOS type annotation.
      if (parentReader.readInt96AsTimeStamp()) {
        return new NullableFixedByteAlignedReaders.NullableFixedBinaryAsTimeStampReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, true, (TimeStampMilliVector) valueVec, schemaElement);
      } else {
        return new NullableFixedByteAlignedReaders.NullableFixedBinaryReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, true, (VarBinaryVector) valueVec, schemaElement);
      }
    }else{
      return new NullableFixedByteAlignedReaders.NullableFixedByteAlignedReader<>(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, valueVec, schemaElement);
    }
  } else {
    switch (columnDescriptor.getType()) {
      case INT32:
        if (convertedType == null) {
          return new NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (IntVector) valueVec, schemaElement);
        }
        switch (convertedType) {
          case DECIMAL:
            return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal9Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (DecimalVector) valueVec, schemaElement);
          case TIME_MILLIS:
            return new NullableFixedByteAlignedReaders.NullableDictionaryTimeReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (TimeMilliVector)valueVec, schemaElement);
          default:
            throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT32");
        }
      case INT64:
        if (convertedType == null) {
          return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (BigIntVector)valueVec, schemaElement);
        }
        switch (convertedType) {
          case DECIMAL:
            return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal18Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (DecimalVector)valueVec, schemaElement);
          case TIMESTAMP_MILLIS:
            return new NullableFixedByteAlignedReaders.NullableDictionaryTimeStampReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (TimeStampMilliVector)valueVec, schemaElement);
          default:
            throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT64");
        }
      case INT96:
        // TODO: check convertedType once parquet support TIMESTAMP_NANOS type annotation.
        if (parentReader.readInt96AsTimeStamp()) {
          return new NullableFixedByteAlignedReaders.NullableFixedBinaryAsTimeStampReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, true, (TimeStampMilliVector) valueVec, schemaElement);
        } else {
          return new NullableFixedByteAlignedReaders.NullableFixedBinaryReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, true, (VarBinaryVector) valueVec, schemaElement);
        }
      case FLOAT:
        return new NullableFixedByteAlignedReaders.NullableDictionaryFloat4Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (Float4Vector)valueVec, schemaElement);
      case DOUBLE:
        return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (Float8Vector)valueVec, schemaElement);
      default:
        throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
    }
  }
}
 

DictionaryTimeStampReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                       ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeStampMilliVector v,
                       SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
 

NullableDictionaryTimeStampReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                               ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeStampMilliVector v,
                               SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
 

NullableFixedBinaryAsTimeStampReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                          ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeStampMilliVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
 

public void set(int position, CompleteType type, ValueVector vector, int offset) {
  if (vector.isNull(offset)) {
    set(position, null);
    return;
  }

  switch (type.toMinorType()) {
    case TINYINT:
    case UINT1:
      setInteger(position, Integer.valueOf((Byte)(vector.getObject(offset))));
      break;
    case SMALLINT:
    case UINT2:
      setInteger(position, Integer.valueOf((Short)(vector.getObject(offset))));
      break;
    case INT:
    case UINT4:
      setInteger(position, (Integer)vector.getObject(offset));
      break;
    case UINT8:
    case BIGINT:
      setLong(position, (Long)(vector.getObject(offset)));
      break;
    case FLOAT4:
      setFloat(position, ((Float)(vector.getObject(offset))));
      break;
    case FLOAT8:
      setDouble(position, ((Double)(vector.getObject(offset))));
      break;
    case BIT:
      setBoolean(position, ((Boolean)(vector.getObject(offset))));
      break;
    case VARBINARY:
      setBytes(position, ((byte[])(vector.getObject(offset))));
      break;
    case DECIMAL9:
    case DECIMAL18:
    case DECIMAL28SPARSE:
    case DECIMAL38SPARSE:
    case DECIMAL28DENSE:
    case DECIMAL38DENSE:
    case DECIMAL:
      setBigDecimal(position, ((BigDecimal)(vector.getObject(offset))));
      break;

    case DATE:
      if (vector instanceof DateMilliVector) {
        setInteger(position, Math.toIntExact(TimeUnit.MILLISECONDS.toDays(((DateMilliVector) vector).get(offset))));
      } else {
        //TODO: needs further tuning
        set(position, null);
      }
      break;
    case TIME:
    case TIMETZ:
    case TIMESTAMPTZ:
    case TIMESTAMP:
    case INTERVAL:
    case INTERVALYEAR:
    case INTERVALDAY:
      if (vector instanceof  TimeStampMilliVector) {
        setLong(position, (((TimeStampMilliVector) vector).get(offset)) * 1000);
      } else {
        //TODO: needs further tuning
        set(position, null);
      }
      break;

    case VARCHAR:
    case FIXEDCHAR:
    case FIXED16CHAR:
    case FIXEDSIZEBINARY:
    case VAR16CHAR:
      setString(position, vector.getObject(offset).toString());
      break;


    case NULL:
    case MONEY:
    case LATE:
    case STRUCT:
    case LIST:
    case GENERIC_OBJECT:
    case UNION:
      throw new IllegalArgumentException("Unsupported type in partition data: " + type.toString());

  }
}
 

public Class<? extends FieldVector> getValueVectorClass(){
  switch (Types.getMinorTypeForArrowType(type)) {
  case UNION:
    return UnionVector.class;
  case STRUCT:
      return StructVector.class;
  case LIST:
      return ListVector.class;
  case NULL:
      return ZeroVector.class;
  case TINYINT:
    return TinyIntVector.class;
  case UINT1:
    return UInt1Vector.class;
  case UINT2:
    return UInt2Vector.class;
  case SMALLINT:
    return SmallIntVector.class;
  case INT:
    return IntVector.class;
  case UINT4:
    return UInt4Vector.class;
  case FLOAT4:
    return Float4Vector.class;
  case INTERVALYEAR:
    return IntervalYearVector.class;
  case TIMEMILLI:
    return TimeMilliVector.class;
  case BIGINT:
    return BigIntVector.class;
  case UINT8:
    return UInt8Vector.class;
  case FLOAT8:
    return Float8Vector.class;
  case DATEMILLI:
    return DateMilliVector.class;
  case TIMESTAMPMILLI:
    return TimeStampMilliVector.class;
  case INTERVALDAY:
    return IntervalDayVector.class;
  case DECIMAL:
    return DecimalVector.class;
  case VARBINARY:
    return VarBinaryVector.class;
  case VARCHAR:
    return VarCharVector.class;
  case BIT:
    return BitVector.class;
  default:
    break;
  }
  throw new UnsupportedOperationException(String.format("Unable to determine vector class for type %s.", type));
}
 

public static FieldVector getNewVector(Field field, BufferAllocator allocator, CallBack callBack) {
  if (field.getType() instanceof ObjectType) {
    return new ObjectVector(field.getName(), allocator);
  }

  MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(field.getType());

  List<Field> children = field.getChildren();

  switch (type) {

  case UNION:
    UnionVector unionVector = new UnionVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      unionVector.initializeChildrenFromFields(children);
    }
    return unionVector;
  case LIST:
    ListVector listVector = new ListVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      listVector.initializeChildrenFromFields(children);
    }
    return listVector;
  case STRUCT:
    StructVector structVector = new StructVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      structVector.initializeChildrenFromFields(children);
    }
    return structVector;

  case NULL:
    return new ZeroVector();
  case TINYINT:
    return new TinyIntVector(field, allocator);
  case UINT1:
    return new UInt1Vector(field, allocator);
  case UINT2:
    return new UInt2Vector(field, allocator);
  case SMALLINT:
    return new SmallIntVector(field, allocator);
  case INT:
    return new IntVector(field, allocator);
  case UINT4:
    return new UInt4Vector(field, allocator);
  case FLOAT4:
    return new Float4Vector(field, allocator);
  case INTERVALYEAR:
    return new IntervalYearVector(field, allocator);
  case TIMEMILLI:
    return new TimeMilliVector(field, allocator);
  case BIGINT:
    return new BigIntVector(field, allocator);
  case UINT8:
    return new UInt8Vector(field, allocator);
  case FLOAT8:
    return new Float8Vector(field, allocator);
  case DATEMILLI:
    return new DateMilliVector(field, allocator);
  case TIMESTAMPMILLI:
    return new TimeStampMilliVector(field, allocator);
  case INTERVALDAY:
    return new IntervalDayVector(field, allocator);
  case DECIMAL:
    return new DecimalVector(field, allocator);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryVector(field.getName(), allocator, WIDTH_ESTIMATE);
  case VARBINARY:
    return new VarBinaryVector(field, allocator);
  case VARCHAR:
    return new VarCharVector(field, allocator);
  case BIT:
    return new BitVector(field, allocator);
  default:
    break;
  }
  // All ValueVector types have been handled.
  throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
}
 

public static Class<?> getValueVectorClass(MinorType type) {
  switch (type) {
  case UNION:
    return UnionVector.class;
  case STRUCT:
    return StructVector.class;
  case LIST:
    return ListVector.class;
  case NULL:
    return ZeroVector.class;
  case TINYINT:
    return TinyIntVector.class;
  case UINT1:
    return UInt1Vector.class;
  case UINT2:
    return UInt2Vector.class;
  case SMALLINT:
    return SmallIntVector.class;
  case INT:
    return IntVector.class;
  case UINT4:
    return UInt4Vector.class;
  case FLOAT4:
    return Float4Vector.class;
  case INTERVALYEAR:
    return IntervalYearVector.class;
  case TIMEMILLI:
    return TimeMilliVector.class;
  case BIGINT:
    return BigIntVector.class;
  case UINT8:
    return UInt8Vector.class;
  case FLOAT8:
    return Float8Vector.class;
  case DATEMILLI:
    return DateMilliVector.class;
  case TIMESTAMPMILLI:
    return TimeStampMilliVector.class;
  case INTERVALDAY:
    return IntervalDayVector.class;
  case DECIMAL:
    return DecimalVector.class;
  case FIXEDSIZEBINARY:
    return FixedSizeBinaryVector.class;
  case VARBINARY:
    return VarBinaryVector.class;
  case VARCHAR:
    return VarCharVector.class;
  case BIT:
    return BitVector.class;
  default:
    break;
  }
  throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type));
}
 

TimestampMilliAccessor(TimeStampMilliVector vector) {
  super(vector);
  this.accessor = vector;
}