Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category#MAP
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category#MAP .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HiveCoercionPolicy.java From presto with Apache License 2.0 | 5 votes |
|
private boolean canCoerceForMap(HiveType fromHiveType, HiveType toHiveType)
{
if (fromHiveType.getCategory() != Category.MAP || toHiveType.getCategory() != Category.MAP) {
return false;
}
HiveType fromKeyType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName());
HiveType fromValueType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName());
HiveType toKeyType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName());
HiveType toValueType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName());
return (fromKeyType.equals(toKeyType) || canCoerce(fromKeyType, toKeyType)) &&
(fromValueType.equals(toValueType) || canCoerce(fromValueType, toValueType));
}
Example 2
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private boolean isSupportedType(Category category) {
return (category == Category.PRIMITIVE ||
category == Category.LIST ||
category == Category.STRUCT ||
category == Category.MAP ||
category == Category.UNION);
}
Example 3
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private boolean isSupportedType(Category category) {
return (category == Category.PRIMITIVE ||
category == Category.LIST ||
category == Category.STRUCT ||
category == Category.MAP ||
category == Category.UNION);
}
Example 4
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Nonnull
public static MapObjectInspector asMapOI(@Nonnull final ObjectInspector oi)
throws UDFArgumentException {
if (oi.getCategory() != Category.MAP) {
throw new UDFArgumentException("Expected Map OI but was: " + oi);
}
return (MapObjectInspector) oi;
}
Example 5
| Source File: MergeMapsUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Override
public MergeMapsEvaluator getEvaluator(TypeInfo[] types) throws SemanticException {
if (types.length != 1) {
throw new UDFArgumentTypeException(types.length - 1,
"One argument is expected but got " + types.length);
}
TypeInfo paramType = types[0];
if (paramType.getCategory() != Category.MAP) {
throw new UDFArgumentTypeException(0, "Only maps supported for now ");
}
return new MergeMapsEvaluator();
}
Example 6
| Source File: SingleLevelArrayMapKeyValuesSchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
}
if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
}
if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
}
if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
Example 7
| Source File: SingleLevelArraySchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo,
Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else {
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else {
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
}
else if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
else {
return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
}
else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
else if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
}
else if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
}
else if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
}
else if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
Example 8
| Source File: MapKeyValuesSchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
else if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
Example 9
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 4 votes |
|
private static boolean searchAllFields(final ObjectInspector rootOI,
final String name,
final int[] childCounts,
SearchResult position
) {
Category category = rootOI.getCategory();
if (category == Category.STRUCT) {
position.index++; // first child is immediately next to parent
StructObjectInspector sOi = (StructObjectInspector) rootOI;
for (StructField sf : sOi.getAllStructFieldRefs()) {
// We depend on the fact that caller takes care of calling current method
// once for each segment in the selected column path. So, we should always get
// searched field as immediate child
if (position.index >= childCounts.length) {
// input schema has more columns than what reader can read
return false;
}
if (sf.getFieldName().equalsIgnoreCase(name)) {
position.oI = sf.getFieldObjectInspector();
return true;
} else {
position.index += childCounts[position.index];
}
}
} else if (category == Category.MAP) {
position.index++; // first child is immediately next to parent
if (position.index >= childCounts.length) {
// input schema has more columns than what reader can read
return false;
}
if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
position.oI = kOi;
return true;
}
position.index += childCounts[position.index];
if (position.index >= childCounts.length) {
// input schema has more columns than what reader can read
return false;
}
if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
position.oI = vOi;
return true;
}
}
return false;
}
Example 10
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 4 votes |
|
private static boolean searchAllFields(final ObjectInspector rootOI,
final String name,
final int[] childCounts,
SearchResult position
) {
Category category = rootOI.getCategory();
if (category == Category.STRUCT) {
position.index++; // first child is immediately next to parent
StructObjectInspector sOi = (StructObjectInspector) rootOI;
for (StructField sf : sOi.getAllStructFieldRefs()) {
// We depend on the fact that caller takes care of calling current method
// once for each segment in the selected column path. So, we should always get
// searched field as immediate child
if (sf.getFieldName().equalsIgnoreCase(name)) {
position.oI = sf.getFieldObjectInspector();
return true;
} else {
if (position.index >= childCounts.length) {
return false;
}
position.index += childCounts[position.index];
}
}
} else if (category == Category.MAP) {
position.index++; // first child is immediately next to parent
if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
position.oI = kOi;
return true;
}
if (position.index >= childCounts.length) {
return false;
}
position.index += childCounts[position.index];
if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
position.oI = vOi;
return true;
}
}
return false;
}
Example 11
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
public static boolean isMapOI(@Nonnull final ObjectInspector oi) {
return oi.getCategory() == Category.MAP;
}
Example 12
| Source File: CassandraColumnSerDe.java From Hive-Cassandra with Apache License 2.0 | 4 votes |
|
/**
* Initialize the cassandra serialization and deserialization parameters from table properties and configuration.
*
* @param job
* @param tbl
* @param serdeName
* @throws SerDeException
*/
@Override
protected void initCassandraSerDeParameters(Configuration job, Properties tbl, String serdeName)
throws SerDeException {
cassandraColumnFamily = getCassandraColumnFamily(tbl);
cassandraColumnNames = parseOrCreateColumnMapping(tbl);
cassandraColumnNamesBytes = new ArrayList<BytesWritable>();
for (String columnName : cassandraColumnNames) {
cassandraColumnNamesBytes.add(new BytesWritable(columnName.getBytes()));
}
iKey = cassandraColumnNames.indexOf(AbstractColumnSerDe.CASSANDRA_KEY_COLUMN);
serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, serdeName);
validatorType = parseOrCreateValidatorType(tbl);
setTableMapping();
if (cassandraColumnNames.size() != serdeParams.getColumnNames().size()) {
throw new SerDeException(serdeName + ": columns has " +
serdeParams.getColumnNames().size() +
" elements while cassandra.columns.mapping has " +
cassandraColumnNames.size() + " elements" +
" (counting the key if implicit)");
}
// we just can make sure that "StandardColumn:" is mapped to MAP<String,?>
for (int i = 0; i < cassandraColumnNames.size(); i++) {
String cassandraColName = cassandraColumnNames.get(i);
if (cassandraColName.endsWith(":")) {
TypeInfo typeInfo = serdeParams.getColumnTypes().get(i);
if ((typeInfo.getCategory() != Category.MAP) ||
(((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getTypeName()
!= Constants.STRING_TYPE_NAME)) {
throw new SerDeException(
serdeName + ": Cassandra column family '"
+ cassandraColName
+ "' should be mapped to map<string,?> but is mapped to "
+ typeInfo.getTypeName());
}
}
}
}
