org.apache.arrow.vector.types.Types.MinorType Java Examples

/**
 * This method is a helper method added for DRILL-951
 * TextRecordReader to call this method to get field names out
 * @return array of field data strings
 */
public String [] getTextOutput () throws ExecutionSetupException {
  if (recordCount == 0 || fieldIndex == -1) {
    return null;
  }

  //Currently only first line header is supported. Return only first record.
  int retSize = fieldIndex+1;
  String [] out = new String [retSize];

  try {
    ListVector listVector = output.addField(new Field(COL_NAME, true, MinorType.LIST.getType(), null), ListVector.class);
    List outputlist = (List) listVector.getObject((int)(recordCount-1));

    for (int i=0; i<retSize; i++){
      out[i] = ((Text) outputlist.get(i)).toString();
    }
    return out;
  } catch (SchemaChangeException e) {
    throw new ExecutionSetupException(e);
  }
}
 

public CustomGeneratorWithSV2(int numRows, BufferAllocator allocator, SelectionVariant variant) {
  Preconditions.checkState(numRows > 0);
  this.numRows = numRows;
  this.stringValues = listOfStrings(numRows);
  this.intValues = randomListOfInts(numRows);
  this.longValues = randomListOfLongs(numRows);
  this.decimalValues = randomListOfDecimals(numRows);
  this.listValues = listOfLists(numRows);
  this.bitValues = randomBits(numRows);
  this.selectedBitSet = new BitSet(numRows);
  this.totalSelectedRows = 0;
  computeSelection(variant);

  this.sv2 = new SelectionVector2(allocator);
  this.container = new VectorContainerWithSV(allocator, sv2);
  this.bitVector = container.addOrGet(BITF);
  this.intVector = container.addOrGet(INTF);
  this.bigIntVector = container.addOrGet(BIGINTF);
  this.decimalVector = container.addOrGet(DECIMALF);
  this.stringVector = container.addOrGet(STRINGF);
  this.listVector = container.addOrGet(LISTF);
  container.buildSchema(SelectionVectorMode.TWO_BYTE);
  listVector.addOrGetVector(FieldType.nullable(MinorType.BIGINT.getType()));
}
 

public static FieldConverter getFieldConverter(final RowBasedRecordWriter recordWriter, final int
  fieldId, final String fieldName, final MinorType type, final  FieldReader reader) {
  switch (type) {
    case NULL:
      return recordWriter.getNewNullConverter(fieldId, fieldName, reader);
    case UNION:
      return recordWriter.getNewUnionConverter(fieldId, fieldName, reader);
    case STRUCT:
      return recordWriter.getNewMapConverter(fieldId, fieldName, reader);
    case LIST:
      return recordWriter.getNewListConverter(fieldId, fieldName, reader);
      <#list vv.types as type>
      <#list type.minor as minor>
      <#assign typeMapping = TypeMappings[minor.class]!{}>
      <#assign supported = typeMapping.supported!true>
      <#if supported>
    case ${minor.class?upper_case}:
      return recordWriter.getNewNullable${minor.class}Converter(fieldId, fieldName, reader);
      </#if>
    </#list >
    </#list >
    default:
      throw new UnsupportedOperationException("unknown type: " + type);
  }
}
 

public void eval() {
  org.apache.arrow.vector.types.Types.MinorType type1;
  if (input1.isSet()) {
    type1 = input1.getMinorType();
  } else {
    type1 = org.apache.arrow.vector.types.Types.MinorType.NULL;
  }
  org.apache.arrow.vector.types.Types.MinorType type2;
  if (input2.isSet()) {
    type2 = input2.getMinorType();
  } else {
    type2 = org.apache.arrow.vector.types.Types.MinorType.NULL;
  }

  out.isSet = 1;
  out.value = com.dremio.exec.expr.fn.impl.UnionFunctions.compareTypes(type1, type2);
}
 

/**
 * Helper method to verify that the batch holder contains valid data including the standard two columns
 * (colBit - BIT, colVarChar - VARCHAR) used in this test class.
 */
private static void verifyBatchHolder(RecordBatchHolder holder, int expStart, int expEnd) {
  assertNotNull(holder);
  assertNotNull(holder.getData());
  assertEquals(expStart, holder.getStart());
  assertEquals(expEnd, holder.getEnd());

  // verify schema
  BatchSchema schema = holder.getData().getContainer().getSchema();
  assertEquals(2, schema.getFieldCount());

  assertEquals("colBit", schema.getColumn(0).getName());
  assertEquals(MinorType.BIT, Types.getMinorTypeForArrowType(schema.getColumn(0).getType()));

  assertEquals("colVarChar", schema.getColumn(1).getName());
  assertEquals(MinorType.VARCHAR, Types.getMinorTypeForArrowType(schema.getColumn(1).getType()));
}
 

/**
 * Gives a type ordering modeled after the behavior of MongoDB
 * Numeric types are first, folowed by string types, followed by binary, then boolean, then date, then timestamp
 * Any other times will be sorted after that
 * @param type
 * @return
 */
private static int getTypeValue(MinorType type) {
  if (TypeCastRules.isNumericType(getMinorTypeFromArrowMinorType(type))) {
    return 0;
  }
  switch (type) {
  case TINYINT:
  case SMALLINT:
  case INT:
  case BIGINT:
  case UINT1:
  case UINT2:
  case UINT4:
  case UINT8:
  case DECIMAL:
  case FLOAT4:
  case FLOAT8:
    return 0;
  case VARCHAR:
    return 1;
  case VARBINARY:
    return 2;
  case BIT:
    return 3;
  case DATEMILLI:
    return 4;
  case TIMESTAMPMILLI:
    return 5;
  default:
    return 6 + type.ordinal();
  }
}
 

public static List<MinorType> getArrowSubtypes(List<TypeProtos.MinorType> subTypes) {
  if (subTypes == null) {
    return null;
  }
  List<MinorType> arrowMinorTypes = new ArrayList<>();
  for (TypeProtos.MinorType minorType : subTypes) {
    arrowMinorTypes.add(getArrowMinorType(minorType));
  }
  return arrowMinorTypes;
}
 

/**
 * Return Dremio minor type for the provided Arrow minor type
 *
 * @param arrowMinorType
 * @return
 */
public static TypeProtos.MinorType getMinorTypeFromArrowMinorType(MinorType arrowMinorType) {
  switch (arrowMinorType) {
  case TIMESTAMPMILLI:
    return TypeProtos.MinorType.TIMESTAMP;
  case TIMESTAMPMILLITZ:
    return TypeProtos.MinorType.TIMESTAMPTZ;
  case TIMEMILLI:
    return TypeProtos.MinorType.TIME;
  case DATEMILLI:
    return TypeProtos.MinorType.DATE;
  default:
    return TypeProtos.MinorType.valueOf(arrowMinorType.name());
  }
}
 

@Override
public ValueHolder getConstantValueHolder(String value, MinorType type, Function<ArrowBuf, ValueHolder> holderInitializer) {
  if (!constantValueHolderCache.containsKey(value)) {
    constantValueHolderCache.put(value, Maps.<MinorType, ValueHolder>newHashMap());
  }

  Map<MinorType, ValueHolder> holdersByType = constantValueHolderCache.get(value);
  ValueHolder valueHolder = holdersByType.get(type);
  if (valueHolder == null) {
    valueHolder = holderInitializer.apply(getManagedBuffer());
    holdersByType.put(type, valueHolder);
  }
  return valueHolder;
}
 

public static FieldConverter getConverter(final RowBasedRecordWriter recordWriter, final int fieldId,
    final String fieldName, final MinorType type, final  FieldReader reader) {
  if (reader.getField().getFieldType().getType().getTypeID() == ArrowTypeID.Union) {
    return recordWriter.getNewUnionConverter(fieldId, fieldName, reader);
  }
  return getFieldConverter(recordWriter,fieldId, fieldName, type, reader);
}
 

public static List<Field> getChildrenForMajorType(MajorType majorType) {
  List<Field> children = new ArrayList<>();
  switch (majorType.getMinorType()) {
  case UNION:
    for (TypeProtos.MinorType minorType : majorType.getSubTypeList()) {
      children.add(Field.nullable("", getArrowMinorType(minorType).getType()));
    }
    return children;
  default:
    return Collections.emptyList();
  }
}
 

public void eval() {
  out.isSet = 1;

  byte[] type;
  if (input.isSet()) {
     type = input.getMinorType().name().getBytes();
  } else {
    type = org.apache.arrow.vector.types.Types.MinorType.NULL.name().getBytes();
  }
  buf = buf.reallocIfNeeded(type.length);
  buf.setBytes(0, type);
  out.buffer = buf;
  out.start = 0;
  out.end = type.length;
}
 

public void eval() {
  if (in.isSet == 1) {
    if (in.reader.getMinorType() != org.apache.arrow.vector.types.Types.MinorType.LIST) {
      throw errorContext.error()
          .message("The input is not a LIST type")
          .build();
    }

    org.apache.arrow.vector.complex.impl.ComplexCopier.copy(in.reader,
        (org.apache.arrow.vector.complex.writer.FieldWriter) out.rootAsList());
  }
}
 

public void eval() {
  out.isSet = 1;
  if (in.isSet == 1) {
    out.value = in.getMinorType() == org.apache.arrow.vector.types.Types.MinorType.LIST ? 1 : 0;
  } else {
    out.value = 0;
  }
}
 

public void eval() {
  if (in.isSet == 1) {
    if (in.reader.getMinorType() != org.apache.arrow.vector.types.Types.MinorType.STRUCT) {
      throw errorContext.error()
          .message("The input is not a STRUCT type")
          .build();
    }

    org.apache.arrow.vector.complex.impl.ComplexCopier.copy(in.reader,
        (org.apache.arrow.vector.complex.writer.FieldWriter) out.rootAsStruct());
  }
}
 

public void eval() {
  out.isSet = 1;
  if (in.isSet == 1) {
    out.value = in.getMinorType() == org.apache.arrow.vector.types.Types.MinorType.STRUCT ? 1 : 0;
  } else {
    out.value = 0;
  }
}
 

@Override
public ValueHolder getConstantValueHolder(String value, MinorType type,
    Function<ArrowBuf, ValueHolder> holderInitializer) {
  if (!constantValueHolderCache.containsKey(value)) {
    constantValueHolderCache.put(value, Maps.<MinorType, ValueHolder>newHashMap());
  }

  Map<MinorType, ValueHolder> holdersByType = constantValueHolderCache.get(value);
  ValueHolder valueHolder = holdersByType.get(type);
  if (valueHolder == null) {
    valueHolder = holderInitializer.apply(getManagedBuffer());
    holdersByType.put(type, valueHolder);
  }
  return valueHolder;
}
 

public BatchSchema getSchema() {
  SchemaBuilder builder = BatchSchema.newBuilder()
    .addField(new Field("n_nationKey", true, MinorType.BIGINT.getType(), null))
    .addField(new Field("n_name", true, MinorType.VARCHAR.getType(), null))
    .addField(new Field("n_regionKey", true, MinorType.BIGINT.getType(), null))
    .addField(new Field("n_comment", true, MinorType.VARCHAR.getType(), null));
  return builder.build();
}
 

public BatchSchema getSchema() {
  SchemaBuilder builder = BatchSchema.newBuilder()
    .addField(new Field("n_nationKey", true, MinorType.BIGINT.getType(), null))
    .addField(new Field("n_name", true, MinorType.VARCHAR.getType(), null))
    .addField(new Field("n_regionKey", true, MinorType.BIGINT.getType(), null))
    .addField(new Field("n_comment", true, MinorType.VARCHAR.getType(), null));
  return builder.build();
}
 

public static MinorType getArrowMinorType(TypeProtos.MinorType minorType) {
  switch (minorType) {
  case LATE:
    return MinorType.NULL;
  case TIMESTAMP:
    return MinorType.TIMESTAMPMILLI;
  case TIME:
    return MinorType.TIMEMILLI;
  case DATE:
    return MinorType.DATEMILLI;
  default:
    return MinorType.valueOf(minorType.name());
  }
}
 

@Test
public void writeAndReadEmptyListVectors() throws Exception {
  try (final BufferAllocator allocator = allocatorRule.newAllocator("test-arrow-file-reader", 0, Long.MAX_VALUE);
       final VectorContainer batchData = createBatch(1, testEmptyListVector(allocator))) {

    final Path basePath = new Path(dateGenFolder.getRoot().getPath());
    final ArrowFileMetadata metadata = writeArrowFile(batchData);
    try (final ArrowFileReader reader =
             new ArrowFileReader(HadoopFileSystem.getLocal(FS_CONF), com.dremio.io.file.Path.of(basePath.toUri()), metadata, allocator)) {

      final List<RecordBatchHolder> batchHolders = reader.read(0, 1);
      assertEquals(1, batchHolders.size());
      assertNotNull(batchHolders.get(0).getData());
      assertEquals(0, batchHolders.get(0).getStart());
      assertEquals(1, batchHolders.get(0).getEnd());

      final BatchSchema schema = batchHolders.get(0).getData().getContainer().getSchema();
      assertEquals(1, schema.getFieldCount());
      assertEquals("emptyListVector", schema.getColumn(0).getName());
      assertEquals(MinorType.LIST, Types.getMinorTypeForArrowType(schema.getColumn(0).getType()));

      final VectorContainer batchContainer = batchHolders.get(0).getData().getContainer();
      assertTrue(Iterators.size(batchContainer.iterator()) == 1);
      for (final VectorWrapper<?> wrapper : batchContainer) {
        assertTrue(wrapper.getValueVector() instanceof ListVector);
        assertTrue(((ListVector) (wrapper.getValueVector())).getDataVector() instanceof NullVector);
      }

      releaseBatches(batchHolders);
    }
  }
}
 

@Override
public DataType extractType(String column, int index){
  Pair<RecordBatchData, Integer> dataBatch = find(index);
  Column columnDef = getColumn(column);
  ValueVector vv = dataBatch.getKey().getVectors().get(columnDef.getIndex());
  if (columnDef.getType() == DataType.MIXED) {
    final int typeValue = ((UnionVector)vv).getTypeValue(dataBatch.getValue());
    return DataTypeUtil.getDataType(getMinorTypeFromArrowMinorType(MinorType.values()[typeValue]));
  }

  return columnDef.getType();
}
 

/**
 * Helper method which converts Hive primitive type to Dremio primitive type
 * @param primitiveTypeInfo
 * @param options
 * @return
 */
private static final MinorType getMinorTypeFromHivePrimitiveTypeInfo(PrimitiveTypeInfo primitiveTypeInfo,
    OptionManager options) {
  switch(primitiveTypeInfo.getPrimitiveCategory()) {
    case BINARY:
      return MinorType.VARBINARY;
    case BOOLEAN:
      return MinorType.BIT;
    case DECIMAL: {

      if (options.getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY).getBoolVal() == false) {
        throw UserException.unsupportedError()
            .message(ExecErrorConstants.DECIMAL_DISABLE_ERR_MSG)
            .build(logger);
      }
      return MinorType.DECIMAL;
    }
    case DOUBLE:
      return MinorType.FLOAT8;
    case FLOAT:
      return MinorType.FLOAT4;
    // TODO (DRILL-2470)
    // Byte and short (tinyint and smallint in SQL types) are currently read as integers
    // as these smaller integer types are not fully supported in Dremio today.
    case SHORT:
    case BYTE:
    case INT:
      return MinorType.INT;
    case LONG:
      return MinorType.BIGINT;
    case STRING:
    case VARCHAR:
    case CHAR:
      return MinorType.VARCHAR;
    case TIMESTAMP:
      return MinorType.TIMESTAMPMILLI;
    case DATE:
      return MinorType.DATEMILLI;
  }
  throwUnsupportedHiveDataTypeError(primitiveTypeInfo.getPrimitiveCategory().toString());
  return null;
}
 

private ValueVector addChild(ValueVector vector, MinorType child) {
  return ((ListVector)vector).addOrGetVector(nullable(child.getType())).getVector();
}
 

public static SqlAccessor getSqlAccessor(ValueVector vector, TimeZone defaultTz) {
  final MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(vector.getField().getType());
  switch(type){
  case UNION:
    return new UnionSqlAccessor((UnionVector) vector);
  case TINYINT:
    return new TinyIntAccessor((TinyIntVector) vector);
  case UINT1:
    return new UInt1Accessor((UInt1Vector) vector);
  case UINT2:
    return new UInt2Accessor((UInt2Vector) vector);
  case SMALLINT:
    return new SmallIntAccessor((SmallIntVector) vector);
  case INT:
    return new IntAccessor((IntVector) vector);
  case UINT4:
    return new UInt4Accessor((UInt4Vector) vector);
  case FLOAT4:
    return new Float4Accessor((Float4Vector) vector);
  case INTERVALYEAR:
    return new IntervalYearAccessor((IntervalYearVector) vector);
  case TIMEMILLI:
    return new TimeMilliAccessor((TimeMilliVector) vector, defaultTz);
  case BIGINT:
    return new BigIntAccessor((BigIntVector) vector);
  case UINT8:
    return new UInt8Accessor((UInt8Vector) vector);
  case FLOAT8:
    return new Float8Accessor((Float8Vector) vector);
  case DATEMILLI:
    return new DateMilliAccessor((DateMilliVector) vector, defaultTz);
  case TIMESTAMPMILLI:
    return new TimeStampMilliAccessor((TimeStampMilliVector) vector, defaultTz);
  case INTERVALDAY:
    return new IntervalDayAccessor((IntervalDayVector) vector);
  case DECIMAL:
    return new DecimalAccessor((DecimalVector) vector);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryAccessor((FixedSizeBinaryVector) vector);
  case VARBINARY:
    return new VarBinaryAccessor((VarBinaryVector) vector);
  case VARCHAR:
    return new VarCharAccessor((VarCharVector) vector);
  case BIT:
    return new BitAccessor((BitVector) vector);
  case STRUCT:
  case LIST:
    return new GenericAccessor(vector);
  }
  throw new UnsupportedOperationException(String.format("Unable to find sql accessor for minor type [%s]", type));
}
 

private static Field bigInt(String name) {
  return new Field(name, true, MinorType.BIGINT.getType(), null);
}
 

private static Field varchar(String name) {
  return new Field(name, true, MinorType.VARCHAR.getType(), null);
}
 

private static Field dbl(String name) {
  return new Field(name, true, MinorType.FLOAT8.getType(), null);
}
 

private static Field map(String name, Field...children) {
  return new Field(name, true, MinorType.STRUCT.getType(), Arrays.asList(children));
}
 

private static Field list(String name, Field inner) {
  return new Field(name, true, MinorType.LIST.getType(),
    Collections.singletonList(inner));
}