Java Code Examples for org.apache.hadoop.hive.serde2.typeinfo.TypeInfo#getCategory()
The following examples show how to use
org.apache.hadoop.hive.serde2.typeinfo.TypeInfo#getCategory() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HiveBucketing.java From presto with Apache License 2.0 | 6 votes |
|
private static boolean containsTimestampBucketedV2(TypeInfo type)
{
switch (type.getCategory()) {
case PRIMITIVE:
return ((PrimitiveTypeInfo) type).getPrimitiveCategory() == TIMESTAMP;
case LIST:
return containsTimestampBucketedV2(((ListTypeInfo) type).getListElementTypeInfo());
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) type;
// Note: we do not check map value type because HiveBucketingV2#hashOfMap hashes map values with v1
return containsTimestampBucketedV2(mapTypeInfo.getMapKeyTypeInfo());
default:
// TODO: support more types, e.g. ROW
throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive category: " + type.getCategory());
}
}
Example 2
| Source File: MDSMapObjectInspector.java From multiple-dimension-spread with Apache License 2.0 | 6 votes |
|
public MDSMapObjectInspector( final MapTypeInfo typeInfo ){
TypeInfo keyTypeInfo = typeInfo.getMapKeyTypeInfo();
if( keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && ( (PrimitiveTypeInfo)keyTypeInfo ).getPrimitiveCategory() == PrimitiveCategory.STRING ){
keyObjectInspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
}
else{
throw new RuntimeException( "Map key type is string only." );
}
valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getMapValueTypeInfo() );
if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
}
else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getMapValueTypeInfo() ) );
}
else{
getField = new NestedGetField();
}
}
Example 3
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 6 votes |
|
public static boolean isNumberTypeInfo(@Nonnull TypeInfo typeInfo) {
if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
return false;
}
switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
case BYTE:
case SHORT:
case INT:
case LONG:
case FLOAT:
case DOUBLE:
case DECIMAL:
return true;
default:
return false;
}
}
Example 4
| Source File: HiveType.java From presto with Apache License 2.0 | 5 votes |
|
public static boolean isSupportedType(TypeInfo typeInfo, StorageFormat storageFormat)
{
switch (typeInfo.getCategory()) {
case PRIMITIVE:
return getPrimitiveType((PrimitiveTypeInfo) typeInfo) != null;
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
return isSupportedType(mapTypeInfo.getMapKeyTypeInfo(), storageFormat) && isSupportedType(mapTypeInfo.getMapValueTypeInfo(), storageFormat);
case LIST:
ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
return isSupportedType(listTypeInfo.getListElementTypeInfo(), storageFormat);
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
return structTypeInfo.getAllStructFieldTypeInfos().stream()
.allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
case UNION:
// This feature (reading uniontypes as structs) has only been verified against Avro and ORC tables. Here's a discussion:
// 1. Avro tables are supported and verified.
// 2. ORC tables are supported and verified.
// 3. The Parquet format doesn't support uniontypes itself so there's no need to add support for it in Presto.
// 4. TODO: RCFile tables are not supported yet.
// 5. TODO: The support for Avro is done in SerDeUtils so it's possible that formats other than Avro are also supported. But verification is needed.
if (storageFormat.getSerDe().equalsIgnoreCase(AVRO.getSerDe()) || storageFormat.getSerDe().equalsIgnoreCase(ORC.getSerDe())) {
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
return unionTypeInfo.getAllUnionObjectTypeInfos().stream()
.allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
}
}
return false;
}
Example 5
| Source File: ColumnVectorAssignorFactory.java From multiple-dimension-spread with Apache License 2.0 | 5 votes |
|
public static IColumnVectorAssignor create( final TypeInfo typeInfo ){
switch ( typeInfo.getCategory() ){
case PRIMITIVE:
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo)typeInfo;
switch( primitiveTypeInfo.getPrimitiveCategory() ){
case STRING:
case BINARY:
return new BytesColumnVectorAssignor();
case BYTE:
return new LongColumnVectorAssignor( BytePrimitiveSetter.getInstance() );
case SHORT:
return new LongColumnVectorAssignor( ShortPrimitiveSetter.getInstance() );
case INT:
return new LongColumnVectorAssignor( IntegerPrimitiveSetter.getInstance() );
case BOOLEAN:
case LONG:
return new LongColumnVectorAssignor( LongPrimitiveSetter.getInstance() );
case FLOAT:
return new DoubleColumnVectorAssignor( FloatPrimitiveSetter.getInstance() );
case DOUBLE:
return new DoubleColumnVectorAssignor( DoublePrimitiveSetter.getInstance() );
case DATE:
case DECIMAL:
case TIMESTAMP:
case VOID:
default:
throw new UnsupportedOperationException( "Unsupport vectorize column " + primitiveTypeInfo.getPrimitiveCategory() );
}
case STRUCT:
case MAP:
case LIST:
case UNION:
default:
throw new UnsupportedOperationException( "Unsupport vectorize column " + typeInfo.getCategory() );
}
}
Example 6
| Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private static boolean isFieldTypeVarchar(FieldSchema hiveField) {
final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
if (typeInfo.getCategory() == Category.PRIMITIVE) {
PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
if (pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR ||
pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR) {
return true;
}
}
return false;
}
Example 7
| Source File: HiveTypeUtil.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Convert Hive data type to a Flink data type.
*
* @param hiveType a Hive data type
* @return the corresponding Flink data type
*/
public static DataType toFlinkType(TypeInfo hiveType) {
checkNotNull(hiveType, "hiveType cannot be null");
switch (hiveType.getCategory()) {
case PRIMITIVE:
return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
case LIST:
ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;
List<String> names = structTypeInfo.getAllStructFieldNames();
List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();
DataTypes.Field[] fields = new DataTypes.Field[names.size()];
for (int i = 0; i < fields.length; i++) {
fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
}
return DataTypes.ROW(fields);
default:
throw new UnsupportedOperationException(
String.format("Flink doesn't support Hive data type %s yet.", hiveType));
}
}
Example 8
| Source File: HiveTypeUtil.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Convert Hive data type to a Flink data type.
*
* @param hiveType a Hive data type
* @return the corresponding Flink data type
*/
public static DataType toFlinkType(TypeInfo hiveType) {
checkNotNull(hiveType, "hiveType cannot be null");
switch (hiveType.getCategory()) {
case PRIMITIVE:
return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
case LIST:
ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;
List<String> names = structTypeInfo.getAllStructFieldNames();
List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();
DataTypes.Field[] fields = new DataTypes.Field[names.size()];
for (int i = 0; i < fields.length; i++) {
fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
}
return DataTypes.ROW(fields);
default:
throw new UnsupportedOperationException(
String.format("Flink doesn't support Hive data type %s yet.", hiveType));
}
}
Example 9
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
public static boolean isFloatingPointTypeInfo(@Nonnull TypeInfo typeInfo) {
if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
return false;
}
switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
case DOUBLE:
case FLOAT:
case DECIMAL:
return true;
default:
return false;
}
}
Example 10
| Source File: MergeMapsUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Override
public MergeMapsEvaluator getEvaluator(TypeInfo[] types) throws SemanticException {
if (types.length != 1) {
throw new UDFArgumentTypeException(types.length - 1,
"One argument is expected but got " + types.length);
}
TypeInfo paramType = types[0];
if (paramType.getCategory() != Category.MAP) {
throw new UDFArgumentTypeException(0, "Only maps supported for now ");
}
return new MergeMapsEvaluator();
}
Example 11
| Source File: HiveInspectors.java From flink with Apache License 2.0 | 5 votes |
|
private static ObjectInspector getObjectInspector(TypeInfo type) {
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
case LIST:
ListTypeInfo listType = (ListTypeInfo) type;
return ObjectInspectorFactory.getStandardListObjectInspector(
getObjectInspector(listType.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
return ObjectInspectorFactory.getStandardMapObjectInspector(
getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structType = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
for (TypeInfo fieldType : fieldTypes) {
fieldInspectors.add(getObjectInspector(fieldType));
}
return ObjectInspectorFactory.getStandardStructObjectInspector(
structType.getAllStructFieldNames(), fieldInspectors);
default:
throw new CatalogException("Unsupported Hive type category " + type.getCategory());
}
}
Example 12
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
public static boolean isIntegerTypeInfo(@Nonnull TypeInfo typeInfo) {
if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
return false;
}
switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
case BYTE:
case SHORT:
case INT:
case LONG:
return true;
default:
return false;
}
}
Example 13
| Source File: OrcLazyObjectInspectorUtils.java From hive-dwrf with Apache License 2.0 | 4 votes |
|
public static ObjectInspector createLazyObjectInspector(TypeInfo info) {
switch (info.getCategory()) {
case PRIMITIVE:
switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
case FLOAT:
return FLOAT_OBJECT_INSPECTOR;
case DOUBLE:
return DOUBLE_OBJECT_INSPECTOR;
case BOOLEAN:
return BOOLEAN_OBJECT_INSPECTOR;
case BYTE:
return BYTE_OBJECT_INSPECTOR;
case SHORT:
return SHORT_OBJECT_INSPECTOR;
case INT:
return INT_OBJECT_INSPECTOR;
case LONG:
return LONG_OBJECT_INSPECTOR;
case BINARY:
return BINARY_OBJECT_INSPECTOR;
case STRING:
return STRING_OBJECT_INSPECTOR;
case TIMESTAMP:
return TIMESTAMP_OBJECT_INSPECTOR;
default:
throw new IllegalArgumentException("Unknown primitive type " +
((PrimitiveTypeInfo) info).getPrimitiveCategory());
}
case STRUCT:
return new OrcLazyStructObjectInspector((StructTypeInfo) info);
case UNION:
return new OrcLazyUnionObjectInspector((UnionTypeInfo) info);
case MAP:
return new OrcLazyMapObjectInspector((MapTypeInfo) info);
case LIST:
return new OrcLazyListObjectInspector((ListTypeInfo) info);
default:
throw new IllegalArgumentException("Unknown type " +
info.getCategory());
}
}
Example 14
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
public static boolean isStringTypeInfo(@Nonnull TypeInfo typeInfo) {
if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
return false;
}
return ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.STRING;
}
Example 15
| Source File: MapKeyValuesSchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
else if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
Example 16
| Source File: OrcLazyObjectInspectorUtils.java From hive-dwrf with Apache License 2.0 | 4 votes |
|
public static ObjectInspector createWritableObjectInspector(TypeInfo info) {
switch (info.getCategory()) {
case PRIMITIVE:
switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
case FLOAT:
return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
case DOUBLE:
return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
case BOOLEAN:
return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
case BYTE:
return PrimitiveObjectInspectorFactory.writableByteObjectInspector;
case SHORT:
return PrimitiveObjectInspectorFactory.writableShortObjectInspector;
case INT:
return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
case LONG:
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
case BINARY:
return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
case STRING:
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
case TIMESTAMP:
return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
default:
throw new IllegalArgumentException("Unknown primitive type " +
((PrimitiveTypeInfo) info).getPrimitiveCategory());
}
case STRUCT:
return new OrcStruct.OrcStructInspector((StructTypeInfo) info);
case UNION:
return new OrcUnion.OrcUnionObjectInspector((UnionTypeInfo) info);
case MAP:
return new OrcStruct.OrcMapObjectInspector((MapTypeInfo) info);
case LIST:
return new OrcStruct.OrcListObjectInspector((ListTypeInfo) info);
default:
throw new IllegalArgumentException("Unknown type " +
info.getCategory());
}
}
Example 17
| Source File: TsFileSerDe.java From incubator-iotdb with Apache License 2.0 | 4 votes |
|
private boolean supportedCategories(TypeInfo ti) {
final ObjectInspector.Category c = ti.getCategory();
return c.equals(ObjectInspector.Category.PRIMITIVE);
}
Example 18
| Source File: HiveSchemaConverter.java From kite with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
static Schema convert(LinkedList<String> path, String name,
TypeInfo type, Collection<String[]> required) {
switch (type.getCategory()) {
case PRIMITIVE:
if (type.getClass() == charClass || type.getClass() == varcharClass) {
// this is required because type name includes length
return Schema.create(Schema.Type.STRING);
}
String typeInfoName = type.getTypeName();
Preconditions.checkArgument(TYPEINFO_TO_TYPE.containsKey(typeInfoName),
"Cannot convert unsupported type: %s", typeInfoName);
return Schema.create(TYPEINFO_TO_TYPE.get(typeInfoName));
case LIST:
return Schema.createArray(optional(convert(path, name,
((ListTypeInfo) type).getListElementTypeInfo(), required)));
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
Preconditions.checkArgument(
"string".equals(mapType.getMapKeyTypeInfo().toString()),
"Non-String map key type: %s", mapType.getMapKeyTypeInfo());
return Schema.createMap(optional(convert(path, name,
mapType.getMapValueTypeInfo(), required)));
case STRUCT:
return convert(path, name, (StructTypeInfo) type, required);
case UNION:
List<TypeInfo> unionTypes = ((UnionTypeInfo) type)
.getAllUnionObjectTypeInfos();
// add NULL so all union types are optional
List<Schema> types = Lists.newArrayList(NULL);
for (int i = 0; i < unionTypes.size(); i += 1) {
// types within unions cannot be required
types.add(convert(
path, name + "_" + i, unionTypes.get(i), NO_REQUIRED_FIELDS));
}
return Schema.createUnion(types);
default:
throw new IllegalArgumentException(
"Unknown TypeInfo category: " + type.getCategory());
}
}
Example 19
| Source File: MDSObjectInspectorFactory.java From multiple-dimension-spread with Apache License 2.0 | 4 votes |
|
public static ObjectInspector craeteObjectInspectorFromTypeInfo( final TypeInfo typeInfo ){
switch ( typeInfo.getCategory() ){
case STRUCT:
return new MDSStructObjectInspector( (StructTypeInfo)typeInfo );
case MAP:
return new MDSMapObjectInspector( (MapTypeInfo)typeInfo );
case LIST:
return new MDSListObjectInspector( (ListTypeInfo)typeInfo );
case UNION:
UnionTypeInfo unionTypeInfo = (UnionTypeInfo)typeInfo;
List<ObjectInspector> unionList = new ArrayList<ObjectInspector>();
for( TypeInfo childTypeInfo : unionTypeInfo.getAllUnionObjectTypeInfos() ){
unionList.add( craeteObjectInspectorFromTypeInfo( childTypeInfo ) );
}
return ObjectInspectorFactory.getStandardUnionObjectInspector( unionList );
case PRIMITIVE:
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo)typeInfo;
switch( primitiveTypeInfo.getPrimitiveCategory() ){
case STRING:
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
case BINARY:
return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
case BOOLEAN:
return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
case BYTE:
return PrimitiveObjectInspectorFactory.writableByteObjectInspector;
case DOUBLE:
return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
case FLOAT:
return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
case INT:
return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
case LONG:
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
case SHORT:
return PrimitiveObjectInspectorFactory.writableShortObjectInspector;
case DATE:
case DECIMAL:
case TIMESTAMP:
case VOID:
default:
throw new UnsupportedOperationException( "Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory() );
}
default:
throw new UnsupportedOperationException( "Unknown category " + typeInfo.getCategory() );
}
}
Example 20
| Source File: HiveBucketingV2.java From presto with Apache License 2.0 | 4 votes |
|
private static int hash(TypeInfo type, Block block, int position)
{
// This function mirrors the behavior of function hashCodeMurmur in
// HIVE-18910 (and following) 7dc47faddb serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
// https://github.com/apache/hive/blob/7dc47faddba9f079bbe2698aaa4d8712e7654f87/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
if (block.isNull(position)) {
return 0;
}
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo typeInfo = (PrimitiveTypeInfo) type;
PrimitiveCategory primitiveCategory = typeInfo.getPrimitiveCategory();
Type prestoType = requireNonNull(HiveType.getPrimitiveType(typeInfo));
switch (primitiveCategory) {
case BOOLEAN:
return prestoType.getBoolean(block, position) ? 1 : 0;
case BYTE:
return SignedBytes.checkedCast(prestoType.getLong(block, position));
case SHORT:
return Murmur3.hash32(bytes(Shorts.checkedCast(prestoType.getLong(block, position))));
case INT:
return Murmur3.hash32(bytes(toIntExact(prestoType.getLong(block, position))));
case LONG:
return Murmur3.hash32(bytes(prestoType.getLong(block, position)));
case FLOAT:
// convert to canonical NaN if necessary
// Sic! we're `floatToIntBits -> cast to float -> floatToRawIntBits` just as it is (implicitly) done in
// https://github.com/apache/hive/blob/7dc47faddba9f079bbe2698aaa4d8712e7654f87/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java#L830
return Murmur3.hash32(bytes(floatToRawIntBits(floatToIntBits(intBitsToFloat(toIntExact(prestoType.getLong(block, position)))))));
case DOUBLE:
// Sic! we're `doubleToLongBits -> cast to double -> doubleToRawLongBits` just as it is (implicitly) done in
// https://github.com/apache/hive/blob/7dc47faddba9f079bbe2698aaa4d8712e7654f87/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java#L836
return Murmur3.hash32(bytes(doubleToRawLongBits(doubleToLongBits(prestoType.getDouble(block, position)))));
case STRING:
return Murmur3.hash32(prestoType.getSlice(block, position).getBytes());
case VARCHAR:
return Murmur3.hash32(prestoType.getSlice(block, position).getBytes());
case DATE:
// day offset from 1970-01-01
return Murmur3.hash32(bytes(toIntExact(prestoType.getLong(block, position))));
// case TIMESTAMP: // TODO (https://github.com/prestosql/presto/issues/1706): support bucketing v2 for timestamp
default:
throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive primitive category: " + primitiveCategory);
}
case LIST:
return hashOfList((ListTypeInfo) type, block.getObject(position, Block.class));
case MAP:
return hashOfMap((MapTypeInfo) type, block.getObject(position, Block.class));
default:
// TODO: support more types, e.g. ROW
throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive category: " + type.getCategory());
}
}
