org.apache.hadoop.hive.ql.udf.generic.GenericUDTF Java Examples

@SuppressWarnings({ "deprecation", "unchecked" })
@Test
public void normalCase() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
  GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
  ObjectInspector resultInspector = func.initialize(inspectors);
  checkResultInspector(resultInspector);
  ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build();
  sketch.update(1, new double[] {1, 2});
  sketch.update(2, new double[] {1, 2});
  MockCollector collector = new MockCollector();
  func.setCollector(collector);
  func.process(new Object[] {new BytesWritable(sketch.toByteArray())});
  Assert.assertEquals(collector.list.size(), 2);
  Assert.assertEquals(((Object[]) collector.list.get(0)).length, 1);
  Assert.assertEquals(((List<Double>) ((Object[]) collector.list.get(0))[0]), Arrays.asList(1.0, 2.0));
  Assert.assertEquals(((List<Double>) ((Object[]) collector.list.get(1))[0]), Arrays.asList(1.0, 2.0));
}
 

@SuppressWarnings("deprecation")
public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[][] rows) throws HiveException {
    final T udtf;
    try {
        udtf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udtf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udtf);
    deserializeObjectByKryo(serialized, clazz);

    udtf.setCollector(new Collector() {
        public void collect(Object input) throws HiveException {
            // noop
        }
    });

    for (Object[] row : rows) {
        udtf.process(row);
    }

    // serialization after processing row
    serialized = serializeObjectByKryo(udtf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udtf.close();
}
 

@SuppressWarnings("deprecation")
public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[][] rows) throws HiveException {
    final T udtf;
    try {
        udtf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udtf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udtf);
    deserializeObjectByKryo(serialized, clazz);

    udtf.setCollector(new Collector() {
        public void collect(Object input) throws HiveException {
            // noop
        }
    });

    for (Object[] row : rows) {
        udtf.process(row);
    }

    // serialization after processing row
    serialized = serializeObjectByKryo(udtf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udtf.close();
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeTooFewInspectors() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeTooManyInspectors() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, stringInspector, stringInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongCategoryArg1() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { structInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongCategoryArg2() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, structInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongTypeArg1() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, stringInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongTypeArg2() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, binaryInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
 public void initializeNoInspectors() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
 public void initializeTooManyInspectors() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, binaryInspector };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentTypeException.class)
 public void initializeWrongCategory() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { structInspector };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 

@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentTypeException.class)
 public void initializeWrongType() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 

private static HiveGenericUDTF init(Class hiveUdfClass, Object[] constantArgs, DataType[] argTypes) throws Exception {
	HiveFunctionWrapper<GenericUDTF> wrapper = new HiveFunctionWrapper(hiveUdfClass.getName());

	HiveGenericUDTF udf = new HiveGenericUDTF(wrapper, hiveShim);

	udf.setArgumentTypesAndConstants(constantArgs, argTypes);
	udf.getHiveResultType(constantArgs, argTypes);

	ObjectInspector[] argumentInspectors = HiveInspectors.toInspectors(hiveShim, constantArgs, argTypes);
	ObjectInspector returnInspector = wrapper.createFunction().initialize(argumentInspectors);

	udf.open(null);

	collector = new TestCollector(returnInspector);
	udf.setCollector(collector);

	return udf;
}
 

public HiveGenericUDTF(HiveFunctionWrapper<GenericUDTF> hiveFunctionWrapper) {
	this.hiveFunctionWrapper = hiveFunctionWrapper;
}
 

public HiveGenericUDTF(HiveFunctionWrapper<GenericUDTF> hiveFunctionWrapper, HiveShim hiveShim) {
	this.hiveFunctionWrapper = hiveFunctionWrapper;
	this.hiveShim = hiveShim;
}
 

/**
 * Create a FunctionDefinition from a Hive function's class name.
 * Called directly by {@link org.apache.flink.table.module.hive.HiveModule}.
 */
public FunctionDefinition createFunctionDefinitionFromHiveFunction(String name, String functionClassName) {
	Class clazz;
	try {
		clazz = Thread.currentThread().getContextClassLoader().loadClass(functionClassName);

		LOG.info("Successfully loaded Hive udf '{}' with class '{}'", name, functionClassName);
	} catch (ClassNotFoundException e) {
		throw new TableException(
			String.format("Failed to initiate an instance of class %s.", functionClassName), e);
	}

	if (UDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveSimpleUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveSimpleUDF(new HiveFunctionWrapper<>(functionClassName), hiveShim)
		);
	} else if (GenericUDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveGenericUDF(new HiveFunctionWrapper<>(functionClassName), hiveShim)
		);
	} else if (GenericUDTF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDTF", name);

		HiveGenericUDTF udtf = new HiveGenericUDTF(new HiveFunctionWrapper<>(functionClassName), hiveShim);

		return new TableFunctionDefinition(
			name,
			udtf,
			GenericTypeInfo.of(Row.class)
		);
	} else if (GenericUDAFResolver2.class.isAssignableFrom(clazz) || UDAF.class.isAssignableFrom(clazz)) {
		HiveGenericUDAF udaf;

		if (GenericUDAFResolver2.class.isAssignableFrom(clazz)) {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF without UDAF bridging", name);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), false, hiveShim);
		} else {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF with UDAF bridging", name);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), true, hiveShim);
		}

		return new AggregateFunctionDefinition(
			name,
			udaf,
			GenericTypeInfo.of(Object.class),
			GenericTypeInfo.of(GenericUDAFEvaluator.AggregationBuffer.class)
		);
	} else {
		throw new IllegalArgumentException(
			String.format("HiveFunctionDefinitionFactory cannot initiate FunctionDefinition for class %s", functionClassName));
	}
}
 

private static HiveGenericUDTF init(Class hiveUdfClass, Object[] constantArgs, DataType[] argTypes) throws Exception {
	HiveFunctionWrapper<GenericUDTF> wrapper = new HiveFunctionWrapper(hiveUdfClass.getName());

	HiveGenericUDTF udf = new HiveGenericUDTF(wrapper);

	udf.setArgumentTypesAndConstants(constantArgs, argTypes);
	udf.getHiveResultType(constantArgs, argTypes);

	ObjectInspector[] argumentInspectors = HiveInspectors.toInspectors(constantArgs, argTypes);
	ObjectInspector returnInspector = wrapper.createFunction().initialize(argumentInspectors);

	udf.open(null);

	collector = new TestCollector(returnInspector);
	udf.setCollector(collector);

	return udf;
}
 

@Override
public FunctionDefinition createFunctionDefinition(String name, CatalogFunction catalogFunction) {
	String functionClassName = catalogFunction.getClassName();

	if (Boolean.valueOf(catalogFunction.getProperties().get(CatalogConfig.IS_GENERIC))) {
		throw new TableException(
			String.format("HiveFunctionDefinitionFactory does not support generic functions %s yet", name));
	}

	Class clazz;
	try {
		clazz = Thread.currentThread().getContextClassLoader().loadClass(functionClassName);

		LOG.info("Successfully loaded Hive udf '{}' with class '{}'", name, functionClassName);
	} catch (ClassNotFoundException e) {
		throw new TableException(
			String.format("Failed to initiate an instance of class %s.", functionClassName), e);
	}

	if (UDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveSimpleUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveSimpleUDF(new HiveFunctionWrapper<>(functionClassName))
		);
	} else if (GenericUDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveGenericUDF(new HiveFunctionWrapper<>(functionClassName))
		);
	} else if (GenericUDTF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDTF", name);

		HiveGenericUDTF udtf = new HiveGenericUDTF(new HiveFunctionWrapper<>(functionClassName));

		return new TableFunctionDefinition(
			name,
			udtf,
			GenericTypeInfo.of(Row.class)
		);
	} else if (GenericUDAFResolver2.class.isAssignableFrom(clazz) || UDAF.class.isAssignableFrom(clazz)) {
		HiveGenericUDAF udaf;

		if (GenericUDAFResolver2.class.isAssignableFrom(clazz)) {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF with no UDAF bridging and Hive version %s",
				name, hiveVersion);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), false, hiveVersion);
		} else {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF with UDAF bridging and Hive version %s",
				name, hiveVersion);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), true, hiveVersion);
		}

		return new AggregateFunctionDefinition(
			name,
			udaf,
			GenericTypeInfo.of(Object.class),
			GenericTypeInfo.of(GenericUDAFEvaluator.AggregationBuffer.class)
		);
	} else {
		throw new IllegalArgumentException(
			String.format("HiveFunctionDefinitionFactory cannot initiate FunctionDefinition for class %s", functionClassName));
	}
}