org.apache.hadoop.hive.serde2.typeinfo.TypeInfo Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.TypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HiveVectorizedReaderSetting.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
public HiveVectorizedReaderSetting( final FileSplit split , final JobConf job , final HiveReaderSetting hiveReaderConfig ) throws IOException{
  this.hiveReaderConfig = hiveReaderConfig;

  rbCtx = Utilities.getVectorizedRowBatchCtx( job );
  partitionValues = new Object[rbCtx.getPartitionColumnCount()];
  if( 0 < partitionValues.length ){
    rbCtx.getPartitionValues( rbCtx, job, split, partitionValues );
  }

  TypeInfo[] typeInfos = rbCtx.getRowColumnTypeInfos();
  columnNames = rbCtx.getRowColumnNames();
  needColumnIds = createNeedColumnId( ColumnProjectionUtils.getReadColumnIDs( job ) );

  projectionColumn = new boolean[columnNames.length];
  assignors = new IColumnVectorAssignor[columnNames.length];
  for( int id : needColumnIds ){
    projectionColumn[id] = true;
    assignors[id] = ColumnVectorAssignorFactory.create( typeInfos[id] );
  }
}
 
Example #2
Source File: MDSSerde.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
private StructTypeInfo getAllReadTypeInfo( final String columnNameProperty , final String columnTypeProperty ){
  ArrayList<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString( columnTypeProperty );
  ArrayList<String> columnNames = new ArrayList<String>();
  if ( columnNameProperty != null && 0 < columnNameProperty.length() ) {
    String[] columnNameArray = columnNameProperty.split(",");
    for( int i = 0 ; i < columnNameArray.length ; i++ ){
      columnNames.add( columnNameArray[i] );
      filedIndexMap.put( columnNameArray[i] , i );
    }
  }
  StructTypeInfo rootType = new StructTypeInfo();

  rootType.setAllStructFieldNames( columnNames );
  rootType.setAllStructFieldTypeInfos( fieldTypes );

  return rootType;
}
 
Example #3
Source File: CobolGroupField.java    From Cobol-to-Hive with Apache License 2.0 6 votes vote down vote up
public List<TypeInfo> getHiveColumnTypes() {
	List<TypeInfo> hiveColumnNames = new ArrayList<TypeInfo>();
	int count = occurs;
	while(count>0) {
		for (CobolField cf : subfields) {
			if (cf.getType().isInGroup(CobolFieldType.Group.ELEMENTARY)) {
				hiveColumnNames.add(cf.getTypeInfo());
			} else {
				hiveColumnNames.addAll(((CobolGroupField) cf)
						.getHiveColumnTypes());
			}
		}
		count--;
	}
	return hiveColumnNames;
}
 
Example #4
Source File: MAPUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new Evaluator();
}
 
Example #5
Source File: HiveOrcSerDeManager.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * Extensible if there's other source-of-truth for fetching schema instead of interacting with HDFS.
 *
 * For purpose of initializing {@link org.apache.hadoop.hive.ql.io.orc.OrcSerde} object, it will require:
 * org.apache.hadoop.hive.serde.serdeConstants#LIST_COLUMNS and
 * org.apache.hadoop.hive.serde.serdeConstants#LIST_COLUMN_TYPES
 *
 */
protected void addSchemaPropertiesHelper(Path path, HiveRegistrationUnit hiveUnit) throws IOException {
  TypeInfo schema = getSchemaFromLatestFile(path, this.fs);
  if (schema instanceof StructTypeInfo) {
    StructTypeInfo structTypeInfo = (StructTypeInfo) schema;
    hiveUnit.setSerDeProp(serdeConstants.LIST_COLUMNS,
        Joiner.on(",").join(structTypeInfo.getAllStructFieldNames()));
    hiveUnit.setSerDeProp(serdeConstants.LIST_COLUMN_TYPES,
        Joiner.on(",").join(
            structTypeInfo.getAllStructFieldTypeInfos().stream().map(x -> x.getTypeName())
                .collect(Collectors.toList())));
  } else {
    // Hive always uses a struct with a field for each of the top-level columns as the root object type.
    // So for here we assume to-be-registered ORC files follow this pattern.
    throw new IllegalStateException("A valid ORC schema should be an instance of struct");
  }
}
 
Example #6
Source File: TestNiFiOrcUtils.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void test_getPrimitiveOrcTypeFromPrimitiveAvroType() throws Exception {
    // Expected ORC types
    TypeInfo[] expectedTypes = {
            TypeInfoCreator.createInt(),
            TypeInfoCreator.createLong(),
            TypeInfoCreator.createBoolean(),
            TypeInfoCreator.createFloat(),
            TypeInfoCreator.createDouble(),
            TypeInfoCreator.createBinary(),
            TypeInfoCreator.createString(),
    };

    Schema testSchema = buildPrimitiveAvroSchema();
    List<Schema.Field> fields = testSchema.getFields();
    for (int i = 0; i < fields.size(); i++) {
        assertEquals(expectedTypes[i], NiFiOrcUtils.getPrimitiveOrcTypeFromPrimitiveAvroType(fields.get(i).schema().getType()));
    }
}
 
Example #7
Source File: AUCUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    if (HiveUtils.isNumberTypeInfo(typeInfo[0]) && HiveUtils.isIntegerTypeInfo(typeInfo[1])) {
        return new ClassificationEvaluator();
    } else {
        ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
        if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
            throw new UDFArgumentTypeException(0,
                "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
        }

        ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
        if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
            throw new UDFArgumentTypeException(1,
                "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
        }

        return new RankingEvaluator();
    }
}
 
Example #8
Source File: PutORC.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public HDFSRecordWriter createHDFSRecordWriter(final ProcessContext context, final FlowFile flowFile, final Configuration conf, final Path path, final RecordSchema schema)
        throws IOException, SchemaNotFoundException {

    final long stripeSize = context.getProperty(STRIPE_SIZE).asDataSize(DataUnit.B).longValue();
    final int bufferSize = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    final CompressionKind compressionType = CompressionKind.valueOf(context.getProperty(COMPRESSION_TYPE).getValue());
    final boolean normalizeForHive = context.getProperty(HIVE_FIELD_NAMES).asBoolean();
    TypeInfo orcSchema = NiFiOrcUtils.getOrcSchema(schema, normalizeForHive);
    final Writer orcWriter = NiFiOrcUtils.createWriter(path, conf, orcSchema, stripeSize, compressionType, bufferSize);
    final String hiveTableName = context.getProperty(HIVE_TABLE_NAME).isSet()
            ? context.getProperty(HIVE_TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue()
            : NiFiOrcUtils.normalizeHiveTableName(schema.getIdentifier().getName().orElse("unknown"));
    final boolean hiveFieldNames = context.getProperty(HIVE_FIELD_NAMES).asBoolean();

    return new ORCHDFSRecordWriter(orcWriter, schema, hiveTableName, hiveFieldNames);
}
 
Example #9
Source File: CobolDeserializer.java    From Cobol-to-Hive with Apache License 2.0 6 votes vote down vote up
private Object deserializeList(String columnName, ListTypeInfo columnType)
		throws RuntimeException {
	int size = Integer.parseInt(rowElements.get(propertiesList.get(fieldNo).get("dcol")));
	
	List<Object> listContents = new ArrayList<Object>();
	TypeInfo ti = columnType.getListElementTypeInfo();
	String tn = columnType.getTypeName();
	rowElements.add("");
	int tempfieldNo = fieldNo, fieldNoList=fieldNo; 
	for (int j = 0; j < size; j++) {
			listContents.add(worker(tn,ti));
			fieldNoList = fieldNo;
			fieldNo = tempfieldNo;
	}
	fieldNo = fieldNoList;
	return listContents;

}
 
Example #10
Source File: OnehotEncodingUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] argTypes)
        throws SemanticException {
    final int numFeatures = argTypes.length;
    if (numFeatures == 0) {
        throw new UDFArgumentException("_FUNC_ requires at least 1 argument");
    }
    for (int i = 0; i < numFeatures; i++) {
        if (argTypes[i] == null) {
            throw new UDFArgumentTypeException(i,
                "Null type is found. Only primitive type arguments are accepted.");
        }
        if (argTypes[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
            throw new UDFArgumentTypeException(i,
                "Only primitive type arguments are accepted but " + argTypes[i].getTypeName()
                        + " was passed as parameter 1.");
        }
    }

    return new GenericUDAFOnehotEncodingEvaluator();
}
 
Example #11
Source File: ExaParquetWriterImpl.java    From hadoop-etl-udfs with MIT License 6 votes vote down vote up
public ExaParquetWriterImpl(final List<String> colNames,
                            final List<TypeInfo> colTypes,
                            final Configuration conf,
                            final Path path,
                            final String compressionType,
                            final ExaIterator exa,
                            final int firstColumnIndex,
                            final List<Integer> dynamicPartitionExaColNums) throws Exception {
    this(HiveSchemaConverter.convert(colNames, colTypes),
            colNames.size(),
            conf,
            path,
            compressionType,
            exa,
            firstColumnIndex,
            dynamicPartitionExaColNums);
}
 
Example #12
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(CharType charType) {
	// Flink and Hive have different length limit for CHAR. Promote it to STRING if it exceeds the limits of
	// Hive and we're told not to check precision. This can be useful when calling Hive UDF to process data.
	if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH || charType.getLength() < 1) {
		if (checkPrecision) {
			throw new CatalogException(
					String.format("HiveCatalog doesn't support char type with length of '%d'. " +
									"The supported length is [%d, %d]",
							charType.getLength(), 1, HiveChar.MAX_CHAR_LENGTH));
		} else {
			return TypeInfoFactory.stringTypeInfo;
		}
	}
	return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
 
Example #13
Source File: TestNiFiOrcUtils.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void test_getOrcField_record() throws Exception {
    final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
    builder.name("int").type().intType().noDefault();
    builder.name("long").type().longType().longDefault(1L);
    builder.name("array").type().array().items().stringType().noDefault();
    Schema testSchema = builder.endRecord();
    TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema);
    assertEquals(
            TypeInfoFactory.getStructTypeInfo(
                    Arrays.asList("int", "long", "array"),
                    Arrays.asList(
                            TypeInfoCreator.createInt(),
                            TypeInfoCreator.createLong(),
                            TypeInfoFactory.getListTypeInfo(TypeInfoCreator.createString()))),
            orcType);
}
 
Example #14
Source File: NiFiOrcUtils.java    From nifi with Apache License 2.0 6 votes vote down vote up
public static TypeInfo getPrimitiveOrcTypeFromPrimitiveAvroType(Schema.Type avroType) throws IllegalArgumentException {
    if (avroType == null) {
        throw new IllegalArgumentException("Avro type is null");
    }
    switch (avroType) {
        case INT:
            return TypeInfoFactory.getPrimitiveTypeInfo("int");
        case LONG:
            return TypeInfoFactory.getPrimitiveTypeInfo("bigint");
        case BOOLEAN:
        case NULL: // ORC has no null type, so just pick the smallest. All values are necessarily null.
            return TypeInfoFactory.getPrimitiveTypeInfo("boolean");
        case BYTES:
            return TypeInfoFactory.getPrimitiveTypeInfo("binary");
        case DOUBLE:
            return TypeInfoFactory.getPrimitiveTypeInfo("double");
        case FLOAT:
            return TypeInfoFactory.getPrimitiveTypeInfo("float");
        case STRING:
            return TypeInfoFactory.getPrimitiveTypeInfo("string");
        default:
            throw new IllegalArgumentException("Avro type " + avroType.getName() + " is not a primitive type");
    }
}
 
Example #15
Source File: OrcStorage.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public void setLocation(String location, Job job) throws IOException {
    Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
    if (!UDFContext.getUDFContext().isFrontend()) {
        typeInfo = (TypeInfo)ObjectSerializer.deserialize(p.getProperty(signature + SchemaSignatureSuffix));
    } else if (typeInfo == null) {
        typeInfo = getTypeInfo(location, job);
    }
    if (typeInfo != null && oi == null) {
        oi = OrcStruct.createObjectInspector(typeInfo);
    }
    if (!UDFContext.getUDFContext().isFrontend()) {
        if (p.getProperty(signature + RequiredColumnsSuffix) != null) {
            mRequiredColumns = (boolean[]) ObjectSerializer.deserialize(p
                    .getProperty(signature + RequiredColumnsSuffix));
            job.getConfiguration().setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
            job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR,
                    getReqiredColumnIdString(mRequiredColumns));
            if (p.getProperty(signature + SearchArgsSuffix) != null) {
                // Bug in setSearchArgument which always expects READ_COLUMN_NAMES_CONF_STR to be set
                job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR,
                        getReqiredColumnNamesString(getSchema(location, job), mRequiredColumns));
            }
        } else if (p.getProperty(signature + SearchArgsSuffix) != null) {
            // Bug in setSearchArgument which always expects READ_COLUMN_NAMES_CONF_STR to be set
            job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR,
                    getReqiredColumnNamesString(getSchema(location, job)));
        }
        if (p.getProperty(signature + SearchArgsSuffix) != null) {
            job.getConfiguration().set(SARG_PUSHDOWN, p.getProperty(signature + SearchArgsSuffix));
        }

    }
    FileInputFormat.setInputPaths(job, location);
}
 
Example #16
Source File: TestSchemaConversion.java    From kite with Apache License 2.0 5 votes vote down vote up
@Test
public void testMap() {
  TypeInfo type = HiveSchemaConverter.convert(SchemaBuilder.builder().map()
      .values().booleanType());

  Assert.assertEquals("Map should be converted to map",
      TypeInfoFactory.getMapTypeInfo(STRING_TYPE_INFO, BOOLEAN_TYPE_INFO),
      type);
}
 
Example #17
Source File: EmoSerDe.java    From emodb with Apache License 2.0 5 votes vote down vote up
/**
 * Deserializes a raw value to the provided type.
 */
private Object deserialize(TypeInfo type, Object rawValue)
        throws SerDeException {
    Object value = null;

    if (rawValue != null) {
        switch (type.getCategory()) {
            case PRIMITIVE:
                value = deserializePrimitive((PrimitiveTypeInfo) type, rawValue);
                break;
            case STRUCT:
                value = deserializeStruct((StructTypeInfo) type, rawValue);
                break;
            case MAP:
                value = deserializeMap((MapTypeInfo) type, rawValue);
                break;
            case LIST:
                value = deserializeList((ListTypeInfo) type, rawValue);
                break;
            case UNION:
                value = deserializeUnion((UnionTypeInfo) type, rawValue);
                break;
        }
    }

    return value;
}
 
Example #18
Source File: LogSerDe.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
public void checkType(List<String> columnNames, List<TypeInfo> columnTypes, int idx, TypeInfo expectedType)
    throws SerDeException {
  if (!columnTypes.get(idx).equals(expectedType)) {
    throw new SerDeException(getClass().getName()
        + " expected type " + expectedType.toString() + ", but column[" + idx + "] named "
        + columnNames.get(idx) + " has type " + columnTypes.get(idx));
  }
}
 
Example #19
Source File: NiFiOrcUtils.java    From nifi with Apache License 2.0 5 votes vote down vote up
public static TypeInfo getOrcSchema(RecordSchema recordSchema, boolean hiveFieldNames) throws IllegalArgumentException {
    List<RecordField> recordFields = recordSchema.getFields();
    if (recordFields != null) {
        List<String> orcFieldNames = new ArrayList<>(recordFields.size());
        List<TypeInfo> orcFields = new ArrayList<>(recordFields.size());
        recordFields.forEach(recordField -> {
            String fieldName = hiveFieldNames ? recordField.getFieldName().toLowerCase() : recordField.getFieldName();
            orcFieldNames.add(fieldName);
            orcFields.add(getOrcField(recordField.getDataType(), hiveFieldNames));
        });
        return TypeInfoFactory.getStructTypeInfo(orcFieldNames, orcFields);
    }
    return null;
}
 
Example #20
Source File: MaxByUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] argTypes)
        throws SemanticException {
    if (argTypes.length != 2) {
        throw new UDFArgumentLengthException(
            "Exactly two arguments are expected: " + argTypes.length);
    }
    ObjectInspector yOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(argTypes[1]);
    if (!ObjectInspectorUtils.compareSupported(yOI)) {
        throw new UDFArgumentTypeException(1,
            "Cannot support comparison of map<> type or complex type containing map<>.");
    }
    return new Evaluator();
}
 
Example #21
Source File: OrcTestToolsTest.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
@Test
public void testSchemaToTypeInfoConversion() throws Exception {
  // Simple non-nested case:
  Schema avroSchema = SchemaBuilder.record("test")
      .fields()
      .name("id")
      .type()
      .intType()
      .noDefault()
      .name("timestamp")
      .type()
      .stringType()
      .noDefault()
      .endRecord();

  TypeInfo orcSchema = OrcTestTools.convertAvroSchemaToOrcSchema(avroSchema);
  String targetOrcSchemaString = "struct<id:int,timestamp:string>";
  Assert.assertEquals(targetOrcSchemaString, orcSchema.toString());

  // Nested case:
  avroSchema = SchemaBuilder.record("nested")
      .fields()
      .name("nestedId")
      .type()
      .array()
      .items()
      .stringType()
      .noDefault()
      .name("timestamp")
      .type()
      .stringType()
      .noDefault()
      .endRecord();
  orcSchema = OrcTestTools.convertAvroSchemaToOrcSchema(avroSchema);
  TypeDescription targetTypeDescription = TypeDescription.createStruct()
      .addField("nestedId", TypeDescription.createList(TypeDescription.createString()))
      .addField("timestamp", TypeDescription.createString());
  Assert.assertEquals(orcSchema.toString().toLowerCase(), targetTypeDescription.toString().toLowerCase());
}
 
Example #22
Source File: HiveSimpleUDF.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void openInternal() {
	LOG.info("Opening HiveSimpleUDF as '{}'", hiveFunctionWrapper.getClassName());

	function = hiveFunctionWrapper.createFunction();

	List<TypeInfo> typeInfos = new ArrayList<>();

	for (DataType arg : argTypes) {
		typeInfos.add(HiveTypeUtil.toHiveTypeInfo(arg, false));
	}

	try {
		method = function.getResolver().getEvalMethod(typeInfos);
		returnInspector = ObjectInspectorFactory.getReflectionObjectInspector(method.getGenericReturnType(),
			ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
		ObjectInspector[] argInspectors = new ObjectInspector[typeInfos.size()];

		for (int i = 0; i < argTypes.length; i++) {
			argInspectors[i] = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfos.get(i));
		}

		conversionHelper = new GenericUDFUtils.ConversionHelper(method, argInspectors);
		conversions = new HiveObjectConversion[argInspectors.length];
		for (int i = 0; i < argInspectors.length; i++) {
			conversions[i] = HiveInspectors.getConversion(argInspectors[i], argTypes[i].getLogicalType(), hiveShim);
		}

		allIdentityConverter = Arrays.stream(conversions)
			.allMatch(conv -> conv instanceof IdentityConversion);
	} catch (Exception e) {
		throw new FlinkHiveUDFException(
			String.format("Failed to open HiveSimpleUDF from %s", hiveFunctionWrapper.getClassName()), e);
	}
}
 
Example #23
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TypeInfo visit(MapType mapType) {
	LogicalType keyType  = mapType.getKeyType();
	LogicalType valueType = mapType.getValueType();
	TypeInfo keyTypeInfo = keyType.accept(this);
	TypeInfo valueTypeInfo = valueType.accept(this);
	if (null == keyTypeInfo || null == valueTypeInfo) {
		return defaultMethod(mapType);
	} else {
		return TypeInfoFactory.getMapTypeInfo(keyTypeInfo, valueTypeInfo);
	}
}
 
Example #24
Source File: TestNiFiOrcUtils.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void test_getOrcField_union_optional_type() {
    final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
    builder.name("union").type().unionOf().nullBuilder().endNull().and().booleanType().endUnion().noDefault();
    RecordSchema testSchema = AvroTypeUtil.createSchema(builder.endRecord());
    TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("union").get().getDataType(), false);
    assertEquals(TypeInfoCreator.createBoolean(), orcType);
}
 
Example #25
Source File: MinByUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] argTypes)
        throws SemanticException {
    if (argTypes.length != 2) {
        throw new UDFArgumentLengthException(
            "Exactly two arguments are expected: " + argTypes.length);
    }
    ObjectInspector yOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(argTypes[1]);
    if (!ObjectInspectorUtils.compareSupported(yOI)) {
        throw new UDFArgumentTypeException(1,
            "Cannot support comparison of map<> type or complex type containing map<>.");
    }
    return new Evaluator();
}
 
Example #26
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TypeInfo visit(CharType charType) {
	if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support char type with length of '%d'. " +
							"The maximum length is %d",
							charType.getLength(), HiveChar.MAX_CHAR_LENGTH));
	}
	return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
 
Example #27
Source File: HiveBucketingV1.java    From presto with Apache License 2.0 5 votes vote down vote up
private static int hashOfMap(MapTypeInfo type, Block singleMapBlock)
{
    TypeInfo keyTypeInfo = type.getMapKeyTypeInfo();
    TypeInfo valueTypeInfo = type.getMapValueTypeInfo();
    int result = 0;
    for (int i = 0; i < singleMapBlock.getPositionCount(); i += 2) {
        result += hash(keyTypeInfo, singleMapBlock, i) ^ hash(valueTypeInfo, singleMapBlock, i + 1);
    }
    return result;
}
 
Example #28
Source File: TestSchemaConversion.java    From kite with Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleRecord() {
  TypeInfo type = HiveSchemaConverter.convert(SIMPLE_RECORD);

  Assert.assertTrue("Record should be converted to struct",
      type instanceof StructTypeInfo);
  Assert.assertEquals("Field names should match",
      Lists.newArrayList("id", "name"),
      ((StructTypeInfo) type).getAllStructFieldNames());
  Assert.assertEquals("Field types should match",
      Lists.newArrayList(
          INT_TYPE_INFO,
          STRING_TYPE_INFO),
      ((StructTypeInfo) type).getAllStructFieldTypeInfos());
}
 
Example #29
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TypeInfo visit(VarBinaryType varBinaryType) {
	// Flink's BytesType is defined as VARBINARY(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish BytesType and a VARBINARY(Integer.MAX_VALUE) instance
	// Thus always treat VARBINARY(Integer.MAX_VALUE) as BytesType
	if (varBinaryType.getLength() == VarBinaryType.MAX_LENGTH) {
		return TypeInfoFactory.binaryTypeInfo;
	}
	return defaultMethod(varBinaryType);
}
 
Example #30
Source File: HiveBucketingV1.java    From presto with Apache License 2.0 5 votes vote down vote up
static int getBucketHashCode(List<TypeInfo> types, Page page, int position)
{
    checkArgument(types.size() == page.getChannelCount());
    int result = 0;
    for (int i = 0; i < page.getChannelCount(); i++) {
        int fieldHash = hash(types.get(i), page.getBlock(i), position);
        result = result * 31 + fieldHash;
    }
    return result;
}