org.apache.hive.service.cli.HiveSQLException Java Examples

private void createHiveServer() {
  HiveServer2 server = new HiveServer2();
  server.init(new HiveConf());
  for (Service service : server.getServices()) {
    if (service instanceof CLIService) {
      _client = (CLIService) service;
    }
  }
  Preconditions.checkNotNull(_client, "CLI service not found in local Hive server");
  try {
    _sessionHandle = _client.openSession(null, null, null);
    _functionRegistry = SessionState.getRegistryForWrite();
    // "map_from_entries" UDF is required to create maps with non-primitive key types
    _functionRegistry.registerGenericUDF("map_from_entries", MapFromEntriesWrapper.class);
    // TODO: This is a hack. Hive's public API does not have a way to register an already created GenericUDF object
    // It only accepts a class name after which the parameterless constructor of the class is called to create a
    // GenericUDF object. This does not work for HiveTestStdUDFWrapper as it accepts the UDF classes as parameters.
    // However, Hive has an internal method which does allow passing GenericUDF objects instead of classes.
    _functionRegistryAddFunctionMethod =
        _functionRegistry.getClass().getDeclaredMethod("addFunction", String.class, FunctionInfo.class);
    _functionRegistryAddFunctionMethod.setAccessible(true);
  } catch (HiveSQLException | NoSuchMethodException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Consumes any exceptions generated by the Hive engine to reveal the underlying exception thrown by the UDF
 */
private RuntimeException fetchUnderlyingException(Throwable e) {
  while (true) {
    if (e instanceof HiveSQLException || e instanceof HiveException) {
      e = e.getCause();
    } else if (e instanceof IOException && e.getCause() instanceof HiveException) {
      e = e.getCause();
    } else {
      break;
    }
  }
  if (e instanceof RuntimeException) {
    return (RuntimeException) e;
  } else {
    return new RuntimeException(e);
  }
}
 

@Override
public void assertFunctionCall(String functionCallString, Object expectedOutputData, Object expectedOutputType) {
  String query = "SELECT " + functionCallString;
  try {
    // Execute the SQL statement and fetch the result
    OperationHandle handle = _client.executeStatement(_sessionHandle, query, null);
    if (handle.hasResultSet()) {
      RowSet rowSet = _client.fetchResults(handle);
      if (rowSet.numRows() > 1 || rowSet.numColumns() > 1) {
        throw new RuntimeException(
            "Expected 1 row and 1 column in query output. Received " + rowSet.numRows() + " rows and "
                + rowSet.numColumns() + " columns.\nQuery: \"" + query + "\"");
      }
      Object[] row = rowSet.iterator().next();
      Object result = row[0];

      Assert.assertEquals(result, expectedOutputData, "UDF output does not match");
      // Get the output data type and convert them to TypeInfo to compare
      ColumnDescriptor outputColumnDescriptor = _client.getResultSetMetadata(handle).getColumnDescriptors().get(0);
      Assert.assertEquals(TypeInfoUtils.getTypeInfoFromTypeString(outputColumnDescriptor.getTypeName().toLowerCase()),
          TypeInfoUtils.getTypeInfoFromObjectInspector((ObjectInspector) expectedOutputType),
          "UDF output type does not match");
    } else {
      throw new RuntimeException("Query did not return any rows. Query: \"" + query + "\"");
    }
  } catch (HiveSQLException e) {
    throw fetchUnderlyingException(e);
  }
}
 

@Test(expected = HiveSQLException.class)
public void testCreateWithDuplicates() throws Exception {
    // load the raw data as a native, managed table
    // and then insert its content into the external one

    String localTable = createTable("createsourceduplicate");
    String load = loadData("createsourceduplicate");

    // create external table
    String ddl =
            "CREATE EXTERNAL TABLE createsaveduplicate ("
                    + "id       BIGINT, "
                    + "name     STRING, "
                    + "links    STRUCT<url:STRING, picture:STRING>) "
                    + tableProps(resource("hive-createsave", "data", targetVersion),
                            "'" + ConfigurationOptions.ES_MAPPING_ID + "'='id'",
                            "'" + ConfigurationOptions.ES_WRITE_OPERATION + "'='create'");

    String selectTest = "SELECT s.name, struct(s.url, s.picture) FROM createsourceduplicate s";

    // transfer data
    String insert =
            "INSERT OVERWRITE TABLE createsaveduplicate "
                    + "SELECT s.id, s.name, named_struct('url', s.url, 'picture', s.picture) FROM createsourceduplicate s";

    System.out.println(ddl);
    System.out.println(server.execute(ddl));
    System.out.println(server.execute(localTable));
    System.out.println(server.execute(load));
    System.out.println(server.execute(selectTest));
    System.out.println(server.execute(insert));
}
 

@Test(expected = HiveSQLException.class)
public void testUpdateWithoutUpsert() throws Exception {
    // load the raw data as a native, managed table
    // and then insert its content into the external one

    String localTable = createTable("updatewoupsertsource");
    String load = loadData("updatewoupsertsource");

    // create external table
    String ddl =
            "CREATE EXTERNAL TABLE updatewoupsertsave ("
                    + "id       BIGINT, "
                    + "name     STRING, "
                    + "links    STRUCT<url:STRING, picture:STRING>) "
                    + tableProps(resource("hive-updatewoupsertsave", "data", targetVersion),
                            "'" + ConfigurationOptions.ES_MAPPING_ID + "'='id'",
                            "'" + ConfigurationOptions.ES_WRITE_OPERATION + "'='update'");

    String selectTest = "SELECT s.name, struct(s.url, s.picture) FROM updatewoupsertsource s";

    // transfer data
    String insert =
            "INSERT OVERWRITE TABLE updatewoupsertsave "
                    + "SELECT s.id, s.name, named_struct('url', s.url, 'picture', s.picture) FROM updatewoupsertsource s";

    System.out.println(ddl);
    System.out.println(server.execute(ddl));
    System.out.println(server.execute(localTable));
    System.out.println(server.execute(load));
    System.out.println(server.execute(selectTest));
    System.out.println(server.execute(insert));
}
 

@Test(expected = HiveSQLException.class)
public void testCreateWithDuplicates() throws Exception {
    // load the raw data as a native, managed table
    // and then insert its content into the external one

    String localTable = createTable("jsoncreatesourceduplicate");
    String load = loadData("jsoncreatesourceduplicate");

    // create external table
    String ddl =
            "CREATE EXTERNAL TABLE jsoncreatesaveduplicate ("
                    + "json     STRING) "
                    + tableProps(resource("json-hive-createsave", "data", targetVersion),
                            "'" + ConfigurationOptions.ES_MAPPING_ID + "'='number'",
                            "'" + ConfigurationOptions.ES_WRITE_OPERATION + "'='create'");

    String selectTest = "SELECT s.json FROM jsoncreatesourceduplicate s";

    // transfer data
    String insert =
            "INSERT OVERWRITE TABLE jsoncreatesaveduplicate "
                    + "SELECT s.json FROM jsoncreatesourceduplicate s";

    System.out.println(ddl);
    System.out.println(server.execute(ddl));
    System.out.println(server.execute(localTable));
    System.out.println(server.execute(load));
    System.out.println(server.execute(selectTest));
    System.out.println(server.execute(insert));
}
 

@Test(expected = HiveSQLException.class)
public void testUpdateWithoutUpsert() throws Exception {
    // load the raw data as a native, managed table
    // and then insert its content into the external one

    String localTable = createTable("jsonupdatewoupsertsource");
    String load = loadData("jsonupdatewoupsertsource");

    // create external table
    String ddl =
            "CREATE EXTERNAL TABLE jsonupdatewoupsertsave ("
                    + "json     STRING) "
                    + tableProps(resource("json-hive-updatewoupsertsave", "data", targetVersion),
                            "'" + ConfigurationOptions.ES_MAPPING_ID + "'='number'",
                            "'" + ConfigurationOptions.ES_WRITE_OPERATION + "'='update'");

    String selectTest = "SELECT s.json FROM jsonupdatewoupsertsource s";

    // transfer data
    String insert =
            "INSERT OVERWRITE TABLE jsonupdatewoupsertsave "
                    + "SELECT s.json FROM jsonupdatewoupsertsource s";

    System.out.println(ddl);
    System.out.println(server.execute(ddl));
    System.out.println(server.execute(localTable));
    System.out.println(server.execute(load));
    System.out.println(server.execute(selectTest));
    System.out.println(server.execute(insert));
}
 

private ResultSet executeQueryWithLog(String query) throws Exception {
  ResultSet rs;
  try {
    LOGGER.info("Running " + query);
    rs = statement.executeQuery(query);
    return rs;
  } catch (HiveSQLException ex) {
    LOGGER.error("Privilege exception occurs when running : " + query);
    throw ex;
  }
}
 

/**
 * The session hook for sentry authorization that sets the required session level configuration
 * 1. Setup the sentry hooks -
 *    semantic, exec and filter hooks
 * 2. Set additional config properties required for auth
 *      set HIVE_EXTENDED_ENITITY_CAPTURE = true
 *      set SCRATCHDIRPERMISSION = 700
 * 3. Add sensitive config parameters to the config restrict list so that they can't be overridden by users
 */
@Override
public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
  // Add sentry hooks to the session configuration
  HiveConf sessionConf = sessionHookContext.getSessionConf();

  appendConfVar(sessionConf, ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
      SEMANTIC_HOOK);
  HiveAuthzConf authzConf = HiveAuthzBindingHook.loadAuthzConf(sessionConf);
  String commandWhitelist =
      authzConf.get(HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST,
          HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT);
  sessionConf.setVar(ConfVars.HIVE_SECURITY_COMMAND_WHITELIST, commandWhitelist);
  sessionConf.setVar(ConfVars.SCRATCHDIRPERMISSION, SCRATCH_DIR_PERMISSIONS);
  sessionConf.setBoolVar(ConfVars.HIVE_CAPTURE_TRANSFORM_ENTITY, true);

  // set user name
  sessionConf.set(HiveAuthzConf.HIVE_ACCESS_SUBJECT_NAME, sessionHookContext.getSessionUser());
  sessionConf.set(HiveAuthzConf.HIVE_SENTRY_SUBJECT_NAME, sessionHookContext.getSessionUser());
  sessionConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER,
          "org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook$SentryHiveAuthorizerFactory");

  // Set MR ACLs to session user
  appendConfVar(sessionConf, JobContext.JOB_ACL_VIEW_JOB,
      sessionHookContext.getSessionUser());
  appendConfVar(sessionConf, JobContext.JOB_ACL_MODIFY_JOB,
      sessionHookContext.getSessionUser());

  // setup restrict list
  sessionConf.addToRestrictList(ACCESS_RESTRICT_LIST);
}
 

/**
 * The session hook for sentry authorization that sets the required session level configuration 1.
 * Setup the sentry hooks - semantic, exec and filter hooks 2. Set additional config properties
 * required for auth set HIVE_EXTENDED_ENITITY_CAPTURE = true set SCRATCHDIRPERMISSION = 700 3.
 * Add sensitive config parameters to the config restrict list so that they can't be overridden by
 * users
 */
@Override
public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
  // Add sentry hooks to the session configuration
  HiveConf sessionConf = sessionHookContext.getSessionConf();

  appendConfVar(sessionConf, ConfVars.SEMANTIC_ANALYZER_HOOK.varname, SEMANTIC_HOOK);
  // enable sentry authorization V2
  sessionConf.setBoolean(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, true);
  sessionConf.setBoolean(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, false);
  sessionConf.set(HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER.varname,
      "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator");

  // grant all privileges for table to its owner
  sessionConf.setVar(ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS, "");

  // Enable compiler to capture transform URI referred in the query
  sessionConf.setBoolVar(ConfVars.HIVE_CAPTURE_TRANSFORM_ENTITY, true);

  // set security command list
  HiveAuthzConf authzConf = HiveAuthzBindingHook.loadAuthzConf(sessionConf);
  String commandWhitelist =
      authzConf.get(HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST,
          HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT);
  sessionConf.setVar(ConfVars.HIVE_SECURITY_COMMAND_WHITELIST, commandWhitelist);

  // set additional configuration properties required for auth
  sessionConf.setVar(ConfVars.SCRATCHDIRPERMISSION, SCRATCH_DIR_PERMISSIONS);

  // setup restrict list
  sessionConf.addToRestrictList(ACCESS_RESTRICT_LIST);

  // set user name
  sessionConf.set(HiveAuthzConf.HIVE_ACCESS_SUBJECT_NAME, sessionHookContext.getSessionUser());
  sessionConf.set(HiveAuthzConf.HIVE_SENTRY_SUBJECT_NAME, sessionHookContext.getSessionUser());

  // Set MR ACLs to session user
  appendConfVar(sessionConf, JobContext.JOB_ACL_VIEW_JOB, sessionHookContext.getSessionUser());
  appendConfVar(sessionConf, JobContext.JOB_ACL_MODIFY_JOB, sessionHookContext.getSessionUser());
}
 

public List<Object[]> executeStatement(String hiveql) {
    try {
        OperationHandle handle = client.executeStatement(sessionHandle, hiveql, new HashMap<>());
        List<Object[]> resultSet = new ArrayList<>();
        if (handle.hasResultSet()) {

            /*
             * fetchResults will by default return 100 rows per fetch (hive 14). For big result sets we need to continuously fetch the result set until all
             * rows are fetched.
             */
            RowSet rowSet;
            while ((rowSet = client.fetchResults(handle)) != null && rowSet.numRows() > 0) {
                for (Object[] row : rowSet) {
                    resultSet.add(row.clone());
                }
            }
        }

        LOGGER.debug("ResultSet:\n"
                + Joiner.on("\n").join(Iterables.transform(resultSet, new Function<Object[], String>() {
                    @Nullable
                    @Override
                    public String apply(@Nullable Object[] objects) {
                        return Joiner.on(", ").useForNull("null").join(objects);
                    }
                })));

        return resultSet;
    } catch (HiveSQLException e) {
        throw new IllegalArgumentException("Failed to executeQuery Hive query " + hiveql + ": " + e.getMessage(),
                e);
    }
}
 

@Test(expected = HiveSQLException.class)
public void testInvalidQuery() throws Throwable {
    try {
        container.executeStatement("use foo");
    } catch (IllegalArgumentException e) {
        throw e.getCause();
    }
}
 

@Override
public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
  underlyingHook.run(sessionHookContext);
}
 

@Test(expected = HiveSQLException.class)
public void basicLoadWithNoGoodCandidateField() throws Exception {

    String create = "CREATE EXTERNAL TABLE jsonartistsread" + testInstance + " (refuse INT, garbage INT) "
            + tableProps(resource("json-hive-artists", "data", targetVersion), "'es.output.json' = 'true'");

    String select = "SELECT * FROM jsonartistsread" + testInstance;

    server.execute(create);
    server.execute(select);

    fail("Should have broken because there are no String fields in the table schema to place the JSON data.");
}