org.apache.hadoop.hive.metastore.api.Function Java Examples

public static Function convertFunction(final String dbName,
                                       final com.amazonaws.services.glue.model.UserDefinedFunction catalogFunction) {
  if (catalogFunction ==  null) {
    return null;
  }
  Function hiveFunction = new Function();
  hiveFunction.setClassName(catalogFunction.getClassName());
  hiveFunction.setCreateTime((int)(catalogFunction.getCreateTime().getTime() / 1000));
  hiveFunction.setDbName(dbName);
  hiveFunction.setFunctionName(catalogFunction.getFunctionName());
  hiveFunction.setFunctionType(FunctionType.JAVA);
  hiveFunction.setOwnerName(catalogFunction.getOwnerName());
  hiveFunction.setOwnerType(convertPrincipalType(com.amazonaws.services.glue.model.PrincipalType.fromValue(catalogFunction.getOwnerType())));
  hiveFunction.setResourceUris(convertResourceUriList(catalogFunction.getResourceUris()));
  return hiveFunction;
}
 

private static Function instantiateHiveFunction(ObjectPath functionPath, CatalogFunction function) {

		boolean isGeneric = Boolean.valueOf(function.getProperties().get(CatalogConfig.IS_GENERIC));

		// Hive Function does not have properties map
		// thus, use a prefix in class name to distinguish Flink and Hive functions
		String functionClassName = isGeneric ?
			FLINK_FUNCTION_PREFIX + function.getClassName() :
			function.getClassName();

		return new Function(
			// due to https://issues.apache.org/jira/browse/HIVE-22053, we have to normalize function name ourselves
			HiveStringUtils.normalizeIdentifier(functionPath.getObjectName()),
			functionPath.getDatabaseName(),
			functionClassName,
			null,			// Owner name
			PrincipalType.GROUP,	// Temporarily set to GROUP type because it's required by Hive. May change later
			(int) (System.currentTimeMillis() / 1000),
			FunctionType.JAVA,		// FunctionType only has JAVA now
			new ArrayList<>()		// Resource URIs
		);
	}
 

@Override
public Function getFunction(IMetaStoreClient client, String dbName, String functionName) throws NoSuchObjectException, TException {
	try {
		// hive-1.x doesn't throw NoSuchObjectException if function doesn't exist, instead it throws a MetaException
		return client.getFunction(dbName, functionName);
	} catch (MetaException e) {
		// need to check the cause and message of this MetaException to decide whether it should actually be a NoSuchObjectException
		if (e.getCause() instanceof NoSuchObjectException) {
			throw (NoSuchObjectException) e.getCause();
		}
		if (e.getMessage().startsWith(NoSuchObjectException.class.getSimpleName())) {
			throw new NoSuchObjectException(e.getMessage());
		}
		throw e;
	}
}
 

@Test
public void callFunctionsNotSet() throws Exception {
  GetAllFunctionsRequest request = new GetAllFunctionsRequest(mapping);
  GetAllFunctionsResponse response = new GetAllFunctionsResponse();

  when(mapping.getClient()).thenReturn(client);
  when(mapping.getClient().get_all_functions()).thenReturn(response);
  List<GetAllFunctionsResponse> responses = request.call();
  assertThat(responses.size(), is(1));
  assertThat(responses.get(0), is(response));
  verify(mapping, never()).transformOutboundFunction(any(Function.class));
}
 

@Override
public List<GetAllFunctionsResponse> call() throws Exception {
  GetAllFunctionsResponse response = mapping.getClient().get_all_functions();
  if (response.getFunctions() != null) {
    for (Function function : response.getFunctions()) {
      mapping.transformOutboundFunction(function);
    }
  }
  return Collections.singletonList(response);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void alter_function(String dbName, String funcName, Function newFunc)
    throws InvalidOperationException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(dbName);
  mapping.checkWritePermissions(newFunc.getDbName());
  mapping
      .getClient()
      .alter_function(mapping.transformInboundDatabaseName(dbName), funcName,
          mapping.transformInboundFunction(newFunc));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Function get_function(String dbName, String funcName) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(dbName);
  return mapping
      .transformOutboundFunction(
          mapping.getClient().get_function(mapping.transformInboundDatabaseName(dbName), funcName));
}
 

@Test
public void transformOutboundFunction() throws Exception {
  Function function = new Function();
  function.setDbName(DB_NAME);
  Function result = databaseMapping.transformOutboundFunction(function);
  assertThat(result, is(sameInstance(function)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 

@Test
public void transformInboundFunction() throws Exception {
  Function function = new Function();
  function.setDbName(DB_NAME);
  Function result = databaseMapping.transformInboundFunction(function);
  assertThat(result, is(sameInstance(function)));
  assertThat(result.getDbName(), is(IN_DB_NAME));
}
 

@Test
public void call() throws Exception {
  GetAllFunctionsRequest request = new GetAllFunctionsRequest(mapping);
  GetAllFunctionsResponse response = new GetAllFunctionsResponse();
  Function function = newFunction("db", "fn1");
  response.addToFunctions(function);

  when(mapping.getClient()).thenReturn(client);
  when(mapping.getClient().get_all_functions()).thenReturn(response);
  List<GetAllFunctionsResponse> responses = request.call();
  assertThat(responses.size(), is(1));
  assertThat(responses.get(0), is(response));
  verify(mapping).transformOutboundFunction(function);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}
 

@Test
public void callFunctionsEmpty() throws Exception {
  // Not 100% sure if this actually can happen but we cover it anyway.
  GetAllFunctionsRequest request = new GetAllFunctionsRequest(mapping);
  GetAllFunctionsResponse response = new GetAllFunctionsResponse();
  response.setFunctions(Collections.emptyList());

  when(mapping.getClient()).thenReturn(client);
  when(mapping.getClient().get_all_functions()).thenReturn(response);
  List<GetAllFunctionsResponse> responses = request.call();
  assertThat(responses.size(), is(1));
  assertThat(responses.get(0), is(response));
  verify(mapping, never()).transformOutboundFunction(any(Function.class));
}
 

public static Function newFunction(String databaseName, String functionName) {
  List<ResourceUri> resourceUris = Lists
      .newArrayList(new ResourceUri(ResourceType.JAR, "hdfs://path/to/my/jar/my.jar"));
  Function function = new Function(functionName, databaseName, "com.hotels.hive.FN", "hadoop", PrincipalType.USER, 0,
      FunctionType.JAVA, resourceUris);
  return function;
}
 

@Test
public void create_function() throws TException {
  Function function = new Function();
  function.setDbName(DB_P);
  Function inboundFunction = new Function();
  when(primaryMapping.transformInboundFunction(function)).thenReturn(inboundFunction);
  handler.create_function(function);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).create_function(inboundFunction);
}
 

@Test
public void alter_function() throws TException {
  Function newFunc = new Function();
  newFunc.setDbName(DB_P);
  Function inboundFunction = new Function();
  when(primaryMapping.transformInboundFunction(newFunc)).thenReturn(inboundFunction);
  handler.alter_function(DB_P, "function", newFunc);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_function(DB_P, "function", inboundFunction);
}
 

@Test
public void get_function() throws TException {
  Function fromClient = new Function();
  Function expected = new Function();
  when(primaryClient.get_function(DB_P, "funcName")).thenReturn(fromClient);
  when(primaryMapping.transformOutboundFunction(fromClient)).thenReturn(expected);
  Function result = handler.get_function(DB_P, "funcName");
  assertThat(result, is(expected));
}
 

public Function getFunction(String databaseName, String functionName) throws MetaException, TException {
	try {
		// Hive may not throw NoSuchObjectException if function doesn't exist, instead it throws a MetaException
		return client.getFunction(databaseName, functionName);
	} catch (MetaException e) {
		// need to check the cause and message of this MetaException to decide whether it should actually be a NoSuchObjectException
		if (e.getCause() instanceof NoSuchObjectException) {
			throw (NoSuchObjectException) e.getCause();
		}
		if (e.getMessage().startsWith(NoSuchObjectException.class.getSimpleName())) {
			throw new NoSuchObjectException(e.getMessage());
		}
		throw e;
	}
}
 

private static Function instantiateHiveFunction(ObjectPath functionPath, CatalogFunction function) {

		boolean isGeneric = function.isGeneric();

		// Hive Function does not have properties map
		// thus, use a prefix in class name to distinguish Flink and Hive functions
		String functionClassName;
		if (function.getFunctionLanguage().equals(FunctionLanguage.JAVA)) {
			functionClassName = isGeneric ?
				FLINK_FUNCTION_PREFIX + function.getClassName() :
				function.getClassName();
		} else if (function.getFunctionLanguage().equals(FunctionLanguage.PYTHON)) {
			functionClassName = FLINK_PYTHON_FUNCTION_PREFIX + function.getClassName();
		} else {
			throw new UnsupportedOperationException("HiveCatalog supports only creating" +
				" JAVA or PYTHON based function for now");
		}

		return new Function(
			// due to https://issues.apache.org/jira/browse/HIVE-22053, we have to normalize function name ourselves
			functionPath.getObjectName().trim().toLowerCase(),
			functionPath.getDatabaseName(),
			functionClassName,
			null,			// Owner name
			PrincipalType.GROUP,	// Temporarily set to GROUP type because it's required by Hive. May change later
			(int) (System.currentTimeMillis() / 1000),
			FunctionType.JAVA,		// FunctionType only has JAVA now
			new ArrayList<>()		// Resource URIs
		);
	}
 

public static com.amazonaws.services.glue.model.UserDefinedFunction convertFunction(final Function hiveFunction) {
  if (hiveFunction == null ){
    return null;
  }
  com.amazonaws.services.glue.model.UserDefinedFunction catalogFunction = new com.amazonaws.services.glue.model.UserDefinedFunction();
  catalogFunction.setClassName(hiveFunction.getClassName());
  catalogFunction.setFunctionName(hiveFunction.getFunctionName());
  catalogFunction.setCreateTime(new Date((long) (hiveFunction.getCreateTime()) * 1000));
  catalogFunction.setOwnerName(hiveFunction.getOwnerName());
  if(hiveFunction.getOwnerType() != null) {
    catalogFunction.setOwnerType(hiveFunction.getOwnerType().name());
  }
  catalogFunction.setResourceUris(covertResourceUriList(hiveFunction.getResourceUris()));
  return catalogFunction;
}
 

public static UserDefinedFunctionInput convertToUserDefinedFunctionInput(Function hiveFunction) {
  UserDefinedFunctionInput functionInput = new UserDefinedFunctionInput();

  functionInput.setClassName(hiveFunction.getClassName());
  functionInput.setFunctionName(hiveFunction.getFunctionName());
  functionInput.setOwnerName(hiveFunction.getOwnerName());
  if(hiveFunction.getOwnerType() != null) {
    functionInput.setOwnerType(hiveFunction.getOwnerType().name());
  }
  functionInput.setResourceUris(HiveToCatalogConverter.covertResourceUriList(hiveFunction.getResourceUris()));
  return functionInput;
}
 

@Test
public void typicalGetAllFunctions() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .withPrimaryPrefix("primary_")
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();
  List<ResourceUri> resourceUris = Lists
      .newArrayList(new ResourceUri(ResourceType.JAR, "hdfs://path/to/my/jar/my.jar"));
  Function localFunction = new Function("fn1", LOCAL_DATABASE, "com.hotels.hive.FN1", "hadoop", PrincipalType.USER, 0,
      FunctionType.JAVA, resourceUris);
  localServer.client().createFunction(localFunction);
  Function remoteFunction = new Function("fn2", REMOTE_DATABASE, "com.hotels.hive.FN1", "hadoop", PrincipalType.USER,
      0, FunctionType.JAVA, resourceUris);
  remoteServer.client().createFunction(remoteFunction);

  GetAllFunctionsResponse allFunctions = proxy.getAllFunctions();
  List<Function> functions = allFunctions.getFunctions();
  assertThat(functions.size(), is(3));
  assertThat(functions.get(0).getFunctionName(), is("fn1"));
  assertThat(functions.get(0).getDbName(), is("primary_" + LOCAL_DATABASE));
  assertThat(functions.get(1).getFunctionName(), is("fn1"));
  assertThat(functions.get(1).getDbName(), is(LOCAL_DATABASE));
  assertThat(functions.get(2).getFunctionName(), is("fn2"));
  assertThat(functions.get(2).getDbName(), is(PREFIXED_REMOTE_DATABASE));
}
 

@Test
public void testGetFunctionValid() throws Exception {
  when(glueClient.getUserDefinedFunction(any(GetUserDefinedFunctionRequest.class)))
          .thenReturn(new GetUserDefinedFunctionResult().withUserDefinedFunction(
              HiveToCatalogConverter.convertFunction(testFunction)));
  Function result = metastoreClient.getFunction(testDB.getName(), testFunction.getFunctionName());
  assertEquals(testFunction, result);
}
 

@Test
public void testListFunctionsValid() throws Exception {
  List<Function> functions = Lists.newArrayList(testFunction);
  when(glueClient.getUserDefinedFunctions(any(GetUserDefinedFunctionsRequest.class)))
          .thenReturn(new GetUserDefinedFunctionsResult().withUserDefinedFunctions(
              HiveToCatalogConverter.convertFunction(testFunction)));
  List<String> result = metastoreClient.getFunctions(testDB.getName(), ".*");
  assertEquals(functions.size(), result.size());
  assertEquals(functions.get(0).getFunctionName(), result.get(0));
}
 

public Function getFunction(String databaseName, String functionName) throws MetaException, TException {
	HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	return hiveShim.getFunction(client, databaseName, functionName);
}
 

public void alterFunction(String databaseName, String functionName, Function function)
		throws InvalidObjectException, MetaException, TException {
	client.alterFunction(databaseName, functionName, function);
}
 

public void createFunction(Function function) throws InvalidObjectException, MetaException, TException {
	client.createFunction(function);
}
 

/**
 * {@inheritDoc}
 */
@Override
public Function get_function(final String dbName, final String funcName) throws TException {
    throw unimplemented("get_function", new Object[]{dbName, funcName});
}
 

/**
 * {@inheritDoc}
 */
@Override
public void create_function(final Function func) throws TException {
    throw unimplemented("create_function", new Object[]{func});
}
 

/**
 * {@inheritDoc}
 */
@Override
public void alter_function(final String dbName, final String funcName, final Function newFunc) throws TException {
    throw unimplemented("alter_function", new Object[]{dbName, funcName, newFunc});
}
 

public void createFunction(Function function) throws InvalidObjectException, MetaException, TException {
	client.createFunction(function);
}