Java Code Examples for org.apache.hadoop.hive.conf.HiveConf#setVar()
The following examples show how to use
org.apache.hadoop.hive.conf.HiveConf#setVar() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveTestUtils.java From flink with Apache License 2.0 | 6 votes |
public static HiveConf createHiveConf() { ClassLoader classLoader = new HiveTestUtils().getClass().getClassLoader(); HiveConf.setHiveSiteLocation(classLoader.getResource(HIVE_SITE_XML)); try { TEMPORARY_FOLDER.create(); String warehouseDir = TEMPORARY_FOLDER.newFolder().getAbsolutePath() + "/metastore_db"; String warehouseUri = String.format(HIVE_WAREHOUSE_URI_FORMAT, warehouseDir); HiveConf hiveConf = new HiveConf(); hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, TEMPORARY_FOLDER.newFolder("hive_warehouse").getAbsolutePath()); hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, warehouseUri); return hiveConf; } catch (IOException e) { throw new CatalogException( "Failed to create test HiveConf to HiveCatalog.", e); } }
Example 2
Source File: RangerHiveAuthorizerBase.java From ranger with Apache License 2.0 | 6 votes |
@Override public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException { LOG.debug("RangerHiveAuthorizerBase.applyAuthorizationConfigPolicy()"); // from SQLStdHiveAccessController.applyAuthorizationConfigPolicy() if (mSessionContext != null && mSessionContext.getClientType() == CLIENT_TYPE.HIVESERVER2) { // Configure PREEXECHOOKS with DisallowTransformHook to disallow transform queries String hooks = hiveConf.getVar(ConfVars.PREEXECHOOKS).trim(); if (hooks.isEmpty()) { hooks = DisallowTransformHook.class.getName(); } else { hooks = hooks + "," + DisallowTransformHook.class.getName(); } hiveConf.setVar(ConfVars.PREEXECHOOKS, hooks); SettableConfigUpdater.setHiveConfWhiteList(hiveConf); } }
Example 3
Source File: HiveTestUtils.java From flink with Apache License 2.0 | 6 votes |
public static HiveConf createHiveConf() { ClassLoader classLoader = new HiveTestUtils().getClass().getClassLoader(); HiveConf.setHiveSiteLocation(classLoader.getResource(HIVE_SITE_XML)); try { TEMPORARY_FOLDER.create(); String warehouseDir = TEMPORARY_FOLDER.newFolder().getAbsolutePath() + "/metastore_db"; String warehouseUri = String.format(HIVE_WAREHOUSE_URI_FORMAT, warehouseDir); HiveConf hiveConf = new HiveConf(); hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, TEMPORARY_FOLDER.newFolder("hive_warehouse").getAbsolutePath()); hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, warehouseUri); return hiveConf; } catch (IOException e) { throw new CatalogException( "Failed to create test HiveConf to HiveCatalog.", e); } }
Example 4
Source File: TestSentryHiveAuthorizationTaskFactory.java From incubator-sentry with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { conf = new HiveConf(); baseDir = Files.createTempDir(); baseDir.setWritable(true, false); conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath()); SessionState.start(conf); conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY, SentryHiveAuthorizationTaskFactoryImpl.class.getName()); db = Mockito.mock(Hive.class); table = new Table(DB, TABLE); partition = new Partition(table); context = new Context(conf); parseDriver = new ParseDriver(); analyzer = new DDLSemanticAnalyzer(conf, db); SessionState.start(conf); Mockito.when(db.getTable(TABLE, false)).thenReturn(table); Mockito.when(db.getPartition(table, new HashMap<String, String>(), false)) .thenReturn(partition); HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator(); auth.setConf(conf); currentUser = auth.getUserName(); }
Example 5
Source File: HiveMetaStoreProxy.java From griffin with Apache License 2.0 | 6 votes |
@Bean public IMetaStoreClient initHiveMetastoreClient() { HiveConf hiveConf = new HiveConf(); hiveConf.set("hive.metastore.local", "false"); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts); hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval); try { client = HiveMetaStoreClient.newSynchronizedClient(new HiveMetaStoreClient(hiveConf)); } catch (Exception e) { LOGGER.error("Failed to connect hive metastore. {}", e); } return client; }
Example 6
Source File: HiveMetaStoreClientFactoryTest.java From incubator-gobblin with Apache License 2.0 | 6 votes |
@Test public void testCreate() throws TException { HiveConf hiveConf = new HiveConf(); HiveMetaStoreClientFactory factory = new HiveMetaStoreClientFactory(hiveConf); // Since we havE a specified hive-site in the classpath, so have to null it out here to proceed the test // The original value it will get if no local hive-site is placed, will be an empty string. hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, ""); hiveConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, ""); IMetaStoreClient msc = factory.create(); String dbName = "test_db"; String description = "test database"; String location = "file:/tmp/" + dbName; Database db = new Database(dbName, description, location, null); msc.dropDatabase(dbName, true, true); msc.createDatabase(db); db = msc.getDatabase(dbName); Assert.assertEquals(db.getName(), dbName); Assert.assertEquals(db.getDescription(), description); Assert.assertEquals(db.getLocationUri(), location); }
Example 7
Source File: HiveTestDataGenerator.java From dremio-oss with Apache License 2.0 | 6 votes |
private HiveConf newHiveConf() { HiveConf conf = new HiveConf(SessionState.class); MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "thrift://localhost:" + port); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); // Metastore needs to be set, and WITH the deprecated key :( // Otherwise, will default to /user/hive/warehouse when trying to create a new database // (database location is now sent by the client to the server...) HiveConf.setVar(conf, ConfVars.METASTOREWAREHOUSE, whDir); conf.set("mapred.job.tracker", "local"); HiveConf.setVar(conf, ConfVars.SCRATCHDIR, getTempDir("scratch_dir")); HiveConf.setVar(conf, ConfVars.LOCALSCRATCHDIR, getTempDir("local_scratch_dir")); HiveConf.setVar(conf, ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); HiveConf.setBoolVar(conf, ConfVars.HIVE_CBO_ENABLED, false); return conf; }
Example 8
Source File: SubmarineMetaStore.java From submarine with Apache License 2.0 | 6 votes |
public SubmarineMetaStore(SubmarineConfiguration submarineConf) { LOG.info("JdbcDriverClassName = {}", submarineConf.getJdbcDriverClassName()); LOG.info("MetastoreJdbcUrl = {}", submarineConf.getMetastoreJdbcUrl()); LOG.info("MetastoreJdbcUserName = {}", submarineConf.getMetastoreJdbcUserName()); LOG.info("MetastoreJdbcPassword = {}", submarineConf.getMetastoreJdbcPassword()); HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, submarineConf.getJdbcDriverClassName()); conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, submarineConf.getMetastoreJdbcUrl()); conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, submarineConf.getMetastoreJdbcUserName()); conf.setVar(HiveConf.ConfVars.METASTOREPWD, submarineConf.getMetastoreJdbcPassword()); // Id can be set to any int value try { this.rs = RawStoreProxy.getProxy(conf, conf, conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 1); } catch (MetaException e) { LOG.error(e.getMessage(), e); } }
Example 9
Source File: StandaloneHiveServerContext.java From HiveRunner with Apache License 2.0 | 5 votes |
protected void configureFileSystem(Path basedir, HiveConf conf) throws IOException { conf.setVar(METASTORECONNECTURLKEY, metaStorageUrl + ";create=true"); createAndSetFolderProperty(METASTOREWAREHOUSE, "warehouse", conf, basedir); createAndSetFolderProperty(LOCALSCRATCHDIR, "localscratchdir", conf, basedir); createAndSetFolderProperty(HIVEHISTORYFILELOC, "tmp", conf, basedir); conf.setBoolVar(HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); createAndSetFolderProperty("hadoop.tmp.dir", "hadooptmp", conf, basedir); createAndSetFolderProperty("test.log.dir", "logs", conf, basedir); /* Tez specific configurations below */ /* Tez will upload a hive-exec.jar to this location. It looks like it will do this only once per test suite so it makes sense to keep this in a central location rather than in the tmp dir of each test. */ File installation_dir = newFolder(basedir, "tez_installation_dir").toFile(); conf.setVar(HiveConf.ConfVars.HIVE_JAR_DIRECTORY, installation_dir.getAbsolutePath()); conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, installation_dir.getAbsolutePath()); }
Example 10
Source File: HiveStreamingDataWriter.java From spark-llap with Apache License 2.0 | 5 votes |
private void createStreamingConnection() throws StreamingException { final StrictDelimitedInputWriter strictDelimitedInputWriter = StrictDelimitedInputWriter.newBuilder() .withFieldDelimiter(',').build(); HiveConf hiveConf = new HiveConf(); hiveConf.set(MetastoreConf.ConfVars.THRIFT_URIS.getHiveName(), metastoreUri); // isolated classloader and shadeprefix are required for reflective instantiation of outputformat class when // creating hive streaming connection (isolated classloader to load different hive versions and shadeprefix for // HIVE-19494) hiveConf.setVar(HiveConf.ConfVars.HIVE_CLASSLOADER_SHADE_PREFIX, "shadehive"); hiveConf.set(MetastoreConf.ConfVars.CATALOG_DEFAULT.getHiveName(), "hive"); // HiveStreamingCredentialProvider requires metastore uri and kerberos principal to obtain delegation token to // talk to secure metastore. When HiveConf is created above, hive-site.xml is used from classpath. This option // is just an override in case if kerberos principal cannot be obtained from hive-site.xml. if (metastoreKrbPrincipal != null) { hiveConf.set(MetastoreConf.ConfVars.KERBEROS_PRINCIPAL.getHiveName(), metastoreKrbPrincipal); } LOG.info("Creating hive streaming connection.."); streamingConnection = HiveStreamingConnection.newBuilder() .withDatabase(db) .withTable(table) .withStaticPartitionValues(partition) .withRecordWriter(strictDelimitedInputWriter) .withHiveConf(hiveConf) .withAgentInfo(jobId + "(" + partitionId + ")") .connect(); streamingConnection.beginTransaction(); LOG.info("{} created hive streaming connection.", streamingConnection.getAgentInfo()); }
Example 11
Source File: TestURI.java From incubator-sentry with Apache License 2.0 | 5 votes |
@BeforeClass public static void setupTestURI() { conf = new HiveConf(); baseDir = Files.createTempDir(); baseDir.setWritable(true, false); conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath()); SessionState.start(conf); }
Example 12
Source File: StandaloneHiveServerContext.java From HiveRunner with Apache License 2.0 | 5 votes |
protected void configureMiscHiveSettings(HiveConf hiveConf) { hiveConf.setBoolVar(HIVESTATSAUTOGATHER, false); // Turn of dependency to calcite library hiveConf.setBoolVar(HIVE_CBO_ENABLED, false); // Disable to get rid of clean up exception when stopping the Session. hiveConf.setBoolVar(HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); hiveConf.setVar(HADOOPBIN, "NO_BIN!"); }
Example 13
Source File: HiveAuthzBindingSessionHook.java From incubator-sentry with Apache License 2.0 | 5 votes |
/** * The session hook for sentry authorization that sets the required session level configuration * 1. Setup the sentry hooks - * semantic, exec and filter hooks * 2. Set additional config properties required for auth * set HIVE_EXTENDED_ENITITY_CAPTURE = true * set SCRATCHDIRPERMISSION = 700 * 3. Add sensitive config parameters to the config restrict list so that they can't be overridden by users */ @Override public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException { // Add sentry hooks to the session configuration HiveConf sessionConf = sessionHookContext.getSessionConf(); appendConfVar(sessionConf, ConfVars.SEMANTIC_ANALYZER_HOOK.varname, SEMANTIC_HOOK); HiveAuthzConf authzConf = HiveAuthzBindingHook.loadAuthzConf(sessionConf); String commandWhitelist = authzConf.get(HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST, HiveAuthzConf.HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT); sessionConf.setVar(ConfVars.HIVE_SECURITY_COMMAND_WHITELIST, commandWhitelist); sessionConf.setVar(ConfVars.SCRATCHDIRPERMISSION, SCRATCH_DIR_PERMISSIONS); sessionConf.setBoolVar(ConfVars.HIVE_CAPTURE_TRANSFORM_ENTITY, true); // set user name sessionConf.set(HiveAuthzConf.HIVE_ACCESS_SUBJECT_NAME, sessionHookContext.getSessionUser()); sessionConf.set(HiveAuthzConf.HIVE_SENTRY_SUBJECT_NAME, sessionHookContext.getSessionUser()); sessionConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook$SentryHiveAuthorizerFactory"); // Set MR ACLs to session user appendConfVar(sessionConf, JobContext.JOB_ACL_VIEW_JOB, sessionHookContext.getSessionUser()); appendConfVar(sessionConf, JobContext.JOB_ACL_MODIFY_JOB, sessionHookContext.getSessionUser()); // setup restrict list sessionConf.addToRestrictList(ACCESS_RESTRICT_LIST); }
Example 14
Source File: HiveTestDataGenerator.java From dremio-oss with Apache License 2.0 | 5 votes |
private HiveTestDataGenerator(final String dbDir, final String whDir) throws Exception { this.dbDir = dbDir; this.whDir = whDir; final HiveConf conf = new HiveConf(); HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false); HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, true); HiveConf.setVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE, whDir); HiveConf.setVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY, String.format("jdbc:derby:;databaseName=%s;create=true", dbDir)); port = MetaStoreUtils.startMetaStore(conf); config = Maps.newHashMap(); config.put(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); }
Example 15
Source File: HiveConfFactory.java From data-highway with Apache License 2.0 | 5 votes |
public HiveConf newInstance(String hiveMetaStoreUris) { synchronized (HiveConf.class) { // The following prevents HiveConf from loading the default hadoop // *-site.xml and hive-site.xml if they're on the classpath. URL hiveSiteLocation = HiveConf.getHiveSiteLocation(); HiveConf.setHiveSiteLocation(null); HiveConf conf = new HiveConf(new Configuration(false), getClass()); HiveConf.setHiveSiteLocation(hiveSiteLocation); conf.setVar(ConfVars.METASTOREURIS, hiveMetaStoreUris); return conf; } }
Example 16
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 4 votes |
private HiveMetaStoreClient getWaggleDanceClient() throws MetaException { HiveConf conf = new HiveConf(); conf.setVar(ConfVars.METASTOREURIS, getWaggleDanceThriftUri()); conf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI, true); return new HiveMetaStoreClient(conf); }
Example 17
Source File: RelaxedSQLStdHiveAccessController.java From beeju with Apache License 2.0 | 4 votes |
@Override public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException { super.applyAuthorizationConfigPolicy(hiveConf); hiveConf.setVar(ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS, ""); }
Example 18
Source File: FlinkStandaloneHiveServerContext.java From flink with Apache License 2.0 | 4 votes |
private void createAndSetFolderProperty( HiveConf.ConfVars var, String folder, HiveConf conf, TemporaryFolder basedir) { conf.setVar(var, newFolder(basedir, folder).getAbsolutePath()); }
Example 19
Source File: FlinkStandaloneHiveServerContext.java From flink with Apache License 2.0 | 4 votes |
private void createAndSetFolderProperty( HiveConf.ConfVars var, String folder, HiveConf conf, TemporaryFolder basedir) { conf.setVar(var, newFolder(basedir, folder).getAbsolutePath()); }
Example 20
Source File: AWSCatalogMetastoreClient.java From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 | 4 votes |
@Override public void setHiveAddedJars(String addedJars) { //taken from HiveMetaStoreClient HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars); }