Java Code Examples for org.apache.hadoop.hive.conf.HiveConf#setVar()

public static HiveConf createHiveConf() {
	ClassLoader classLoader = new HiveTestUtils().getClass().getClassLoader();
	HiveConf.setHiveSiteLocation(classLoader.getResource(HIVE_SITE_XML));

	try {
		TEMPORARY_FOLDER.create();
		String warehouseDir = TEMPORARY_FOLDER.newFolder().getAbsolutePath() + "/metastore_db";
		String warehouseUri = String.format(HIVE_WAREHOUSE_URI_FORMAT, warehouseDir);

		HiveConf hiveConf = new HiveConf();
		hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, TEMPORARY_FOLDER.newFolder("hive_warehouse").getAbsolutePath());
		hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, warehouseUri);
		return hiveConf;
	} catch (IOException e) {
		throw new CatalogException(
			"Failed to create test HiveConf to HiveCatalog.", e);
	}
}
 

@Override
public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException {
	LOG.debug("RangerHiveAuthorizerBase.applyAuthorizationConfigPolicy()");

	// from SQLStdHiveAccessController.applyAuthorizationConfigPolicy()
	if (mSessionContext != null && mSessionContext.getClientType() == CLIENT_TYPE.HIVESERVER2) {
		// Configure PREEXECHOOKS with DisallowTransformHook to disallow transform queries
		String hooks = hiveConf.getVar(ConfVars.PREEXECHOOKS).trim();
		if (hooks.isEmpty()) {
			hooks = DisallowTransformHook.class.getName();
		} else {
			hooks = hooks + "," + DisallowTransformHook.class.getName();
		}

		hiveConf.setVar(ConfVars.PREEXECHOOKS, hooks);

		SettableConfigUpdater.setHiveConfWhiteList(hiveConf);
	}
}
 

public static HiveConf createHiveConf() {
	ClassLoader classLoader = new HiveTestUtils().getClass().getClassLoader();
	HiveConf.setHiveSiteLocation(classLoader.getResource(HIVE_SITE_XML));

	try {
		TEMPORARY_FOLDER.create();
		String warehouseDir = TEMPORARY_FOLDER.newFolder().getAbsolutePath() + "/metastore_db";
		String warehouseUri = String.format(HIVE_WAREHOUSE_URI_FORMAT, warehouseDir);

		HiveConf hiveConf = new HiveConf();
		hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, TEMPORARY_FOLDER.newFolder("hive_warehouse").getAbsolutePath());
		hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, warehouseUri);
		return hiveConf;
	} catch (IOException e) {
		throw new CatalogException(
			"Failed to create test HiveConf to HiveCatalog.", e);
	}
}
 

@Before
public void setup() throws Exception {
  conf = new HiveConf();
  baseDir = Files.createTempDir();
  baseDir.setWritable(true, false);
  conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath());
  SessionState.start(conf);
  conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
      SentryHiveAuthorizationTaskFactoryImpl.class.getName());

  db = Mockito.mock(Hive.class);
  table = new Table(DB, TABLE);
  partition = new Partition(table);
  context = new Context(conf);
  parseDriver = new ParseDriver();
  analyzer = new DDLSemanticAnalyzer(conf, db);
  SessionState.start(conf);
  Mockito.when(db.getTable(TABLE, false)).thenReturn(table);
  Mockito.when(db.getPartition(table, new HashMap<String, String>(), false))
  .thenReturn(partition);

  HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator();
  auth.setConf(conf);
  currentUser = auth.getUserName();

}
 

@Bean
public IMetaStoreClient initHiveMetastoreClient() {
    HiveConf hiveConf = new HiveConf();
    hiveConf.set("hive.metastore.local", "false");
    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES,
        3);
    hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
    hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts);
    hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval);
    try {
        client = HiveMetaStoreClient.newSynchronizedClient(new HiveMetaStoreClient(hiveConf));
    } catch (Exception e) {
        LOGGER.error("Failed to connect hive metastore. {}", e);
    }
    return client;
}
 

@Test
public void testCreate() throws TException {
  HiveConf hiveConf = new HiveConf();
  HiveMetaStoreClientFactory factory = new HiveMetaStoreClientFactory(hiveConf);

  // Since we havE a specified hive-site in the classpath, so have to null it out here to proceed the test
  // The original value it will get if no local hive-site is placed, will be an empty string. 
  hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "");
  hiveConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, "");
  IMetaStoreClient msc = factory.create();

  String dbName = "test_db";
  String description = "test database";
  String location = "file:/tmp/" + dbName;
  Database db = new Database(dbName, description, location, null);

  msc.dropDatabase(dbName, true, true);
  msc.createDatabase(db);
  db = msc.getDatabase(dbName);
  Assert.assertEquals(db.getName(), dbName);
  Assert.assertEquals(db.getDescription(), description);
  Assert.assertEquals(db.getLocationUri(), location);
}
 

private HiveConf newHiveConf() {
  HiveConf conf = new HiveConf(SessionState.class);

  MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "thrift://localhost:" + port);
  conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
  // Metastore needs to be set, and WITH the deprecated key :(
  // Otherwise, will default to /user/hive/warehouse when trying to create a new database
  // (database location is now sent by the client to the server...)
  HiveConf.setVar(conf, ConfVars.METASTOREWAREHOUSE, whDir);
  conf.set("mapred.job.tracker", "local");
  HiveConf.setVar(conf, ConfVars.SCRATCHDIR,  getTempDir("scratch_dir"));
  HiveConf.setVar(conf, ConfVars.LOCALSCRATCHDIR, getTempDir("local_scratch_dir"));
  HiveConf.setVar(conf, ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
  HiveConf.setBoolVar(conf, ConfVars.HIVE_CBO_ENABLED, false);

  return conf;

}
 

public SubmarineMetaStore(SubmarineConfiguration submarineConf) {
  LOG.info("JdbcDriverClassName = {}", submarineConf.getJdbcDriverClassName());
  LOG.info("MetastoreJdbcUrl = {}", submarineConf.getMetastoreJdbcUrl());
  LOG.info("MetastoreJdbcUserName = {}", submarineConf.getMetastoreJdbcUserName());
  LOG.info("MetastoreJdbcPassword = {}", submarineConf.getMetastoreJdbcPassword());

  HiveConf conf = new HiveConf();
  conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, submarineConf.getJdbcDriverClassName());
  conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, submarineConf.getMetastoreJdbcUrl());
  conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, submarineConf.getMetastoreJdbcUserName());
  conf.setVar(HiveConf.ConfVars.METASTOREPWD, submarineConf.getMetastoreJdbcPassword());
  // Id can be set to any int value
  try {
    this.rs = RawStoreProxy.getProxy(conf, conf,
        conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 1);
  } catch (MetaException e) {
    LOG.error(e.getMessage(), e);
  }
}
 

protected void configureFileSystem(Path basedir, HiveConf conf) throws IOException {
    conf.setVar(METASTORECONNECTURLKEY, metaStorageUrl + ";create=true");

    createAndSetFolderProperty(METASTOREWAREHOUSE, "warehouse", conf, basedir);
    createAndSetFolderProperty(LOCALSCRATCHDIR, "localscratchdir", conf, basedir);
    createAndSetFolderProperty(HIVEHISTORYFILELOC, "tmp", conf, basedir);

    conf.setBoolVar(HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true);

    createAndSetFolderProperty("hadoop.tmp.dir", "hadooptmp", conf, basedir);
    createAndSetFolderProperty("test.log.dir", "logs", conf, basedir);



    /*
        Tez specific configurations below
     */
    /*
        Tez will upload a hive-exec.jar to this location.
        It looks like it will do this only once per test suite so it makes sense to keep this in a central location
        rather than in the tmp dir of each test.
     */
    File installation_dir = newFolder(basedir, "tez_installation_dir").toFile();

    conf.setVar(HiveConf.ConfVars.HIVE_JAR_DIRECTORY, installation_dir.getAbsolutePath());
    conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, installation_dir.getAbsolutePath());
}
 

private void createStreamingConnection() throws StreamingException {
  final StrictDelimitedInputWriter strictDelimitedInputWriter = StrictDelimitedInputWriter.newBuilder()
    .withFieldDelimiter(',').build();
  HiveConf hiveConf = new HiveConf();
  hiveConf.set(MetastoreConf.ConfVars.THRIFT_URIS.getHiveName(), metastoreUri);
  // isolated classloader and shadeprefix are required for reflective instantiation of outputformat class when
  // creating hive streaming connection (isolated classloader to load different hive versions and shadeprefix for
  // HIVE-19494)
  hiveConf.setVar(HiveConf.ConfVars.HIVE_CLASSLOADER_SHADE_PREFIX, "shadehive");
  hiveConf.set(MetastoreConf.ConfVars.CATALOG_DEFAULT.getHiveName(), "hive");

  // HiveStreamingCredentialProvider requires metastore uri and kerberos principal to obtain delegation token to
  // talk to secure metastore. When HiveConf is created above, hive-site.xml is used from classpath. This option
  // is just an override in case if kerberos principal cannot be obtained from hive-site.xml.
  if (metastoreKrbPrincipal != null) {
    hiveConf.set(MetastoreConf.ConfVars.KERBEROS_PRINCIPAL.getHiveName(), metastoreKrbPrincipal);
  }

  LOG.info("Creating hive streaming connection..");
  streamingConnection = HiveStreamingConnection.newBuilder()
    .withDatabase(db)
    .withTable(table)
    .withStaticPartitionValues(partition)
    .withRecordWriter(strictDelimitedInputWriter)
    .withHiveConf(hiveConf)
    .withAgentInfo(jobId + "(" + partitionId + ")")
    .connect();
  streamingConnection.beginTransaction();
  LOG.info("{} created hive streaming connection.", streamingConnection.getAgentInfo());
}
 

@BeforeClass
public static void setupTestURI() {
  conf = new HiveConf();
  baseDir = Files.createTempDir();
  baseDir.setWritable(true, false);
  conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath());
  SessionState.start(conf);
}
 

protected void configureMiscHiveSettings(HiveConf hiveConf) {
    hiveConf.setBoolVar(HIVESTATSAUTOGATHER, false);

    // Turn of dependency to calcite library
    hiveConf.setBoolVar(HIVE_CBO_ENABLED, false);

    // Disable to get rid of clean up exception when stopping the Session.
    hiveConf.setBoolVar(HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false);

    hiveConf.setVar(HADOOPBIN, "NO_BIN!");
}
 

/**
 * The session hook for sentry authorization that sets the required session level configuration
 * 1. Setup the sentry hooks -
 *    semantic, exec and filter hooks
 * 2. Set additional config properties required for auth
 *      set HIVE_EXTENDED_ENITITY_CAPTURE = true
 *      set SCRATCHDIRPERMISSION = 700
 * 3. Add sensitive config parameters to the config restrict list so that they can't be overridden by users
 */
@Override
public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
  // Add sentry hooks to the session configuration
  HiveConf sessionConf = sessionHookContext.getSessionConf();

  appendConfVar(sessionConf, ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
      SEMANTIC_HOOK);
  HiveAuthzConf authzConf = HiveAuthzBindingHook.loadAuthzConf(sessionConf);
  String commandWhitelist =
      authzConf.get(HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST,
          HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT);
  sessionConf.setVar(ConfVars.HIVE_SECURITY_COMMAND_WHITELIST, commandWhitelist);
  sessionConf.setVar(ConfVars.SCRATCHDIRPERMISSION, SCRATCH_DIR_PERMISSIONS);
  sessionConf.setBoolVar(ConfVars.HIVE_CAPTURE_TRANSFORM_ENTITY, true);

  // set user name
  sessionConf.set(HiveAuthzConf.HIVE_ACCESS_SUBJECT_NAME, sessionHookContext.getSessionUser());
  sessionConf.set(HiveAuthzConf.HIVE_SENTRY_SUBJECT_NAME, sessionHookContext.getSessionUser());
  sessionConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER,
          "org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook$SentryHiveAuthorizerFactory");

  // Set MR ACLs to session user
  appendConfVar(sessionConf, JobContext.JOB_ACL_VIEW_JOB,
      sessionHookContext.getSessionUser());
  appendConfVar(sessionConf, JobContext.JOB_ACL_MODIFY_JOB,
      sessionHookContext.getSessionUser());

  // setup restrict list
  sessionConf.addToRestrictList(ACCESS_RESTRICT_LIST);
}
 

private HiveTestDataGenerator(final String dbDir, final String whDir) throws Exception {
  this.dbDir = dbDir;
  this.whDir = whDir;

  final HiveConf conf = new HiveConf();

  HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false);
  HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, true);
  HiveConf.setVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE, whDir);
  HiveConf.setVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY, String.format("jdbc:derby:;databaseName=%s;create=true", dbDir));
  port = MetaStoreUtils.startMetaStore(conf);

  config = Maps.newHashMap();
  config.put(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
}
 

public HiveConf newInstance(String hiveMetaStoreUris) {
  synchronized (HiveConf.class) {
    // The following prevents HiveConf from loading the default hadoop
    // *-site.xml and hive-site.xml if they're on the classpath.
    URL hiveSiteLocation = HiveConf.getHiveSiteLocation();
    HiveConf.setHiveSiteLocation(null);
    HiveConf conf = new HiveConf(new Configuration(false), getClass());
    HiveConf.setHiveSiteLocation(hiveSiteLocation);
    conf.setVar(ConfVars.METASTOREURIS, hiveMetaStoreUris);
    return conf;
  }
}
 

private HiveMetaStoreClient getWaggleDanceClient() throws MetaException {
  HiveConf conf = new HiveConf();
  conf.setVar(ConfVars.METASTOREURIS, getWaggleDanceThriftUri());
  conf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI, true);
  return new HiveMetaStoreClient(conf);
}
 

@Override
public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException {
  super.applyAuthorizationConfigPolicy(hiveConf);
  hiveConf.setVar(ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS, "");
}
 

private void createAndSetFolderProperty(
		HiveConf.ConfVars var, String folder, HiveConf conf,
		TemporaryFolder basedir) {
	conf.setVar(var, newFolder(basedir, folder).getAbsolutePath());
}
 

private void createAndSetFolderProperty(
		HiveConf.ConfVars var, String folder, HiveConf conf,
		TemporaryFolder basedir) {
	conf.setVar(var, newFolder(basedir, folder).getAbsolutePath());
}
 

@Override
public void setHiveAddedJars(String addedJars) {
  //taken from HiveMetaStoreClient
  HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars);
}