org.apache.hadoop.hive.ql.session.SessionState Java Examples

public void verifyLocalQuery(String queryStr) throws Exception {
  // setup Hive driver
  SessionState session = new SessionState(getHiveConf());
  SessionState.start(session);
  Driver driver = new Driver(session.getConf(), getUser());

  // compile the query
  CommandProcessorResponse compilerStatus = driver
      .compileAndRespond(queryStr);
  if (compilerStatus.getResponseCode() != 0) {
    String errMsg = compilerStatus.getErrorMessage();
    if (errMsg.contains(HiveAuthzConf.HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE)) {
      printMissingPerms(getHiveConf().get(
          HiveAuthzConf.HIVE_SENTRY_AUTH_ERRORS));
    }
    throw new SemanticException("Compilation error: "
        + compilerStatus.getErrorMessage());
  }
  driver.close();
  System.out
      .println("User " + getUser() + " has privileges to run the query");
}
 

private static void generateHiveTestData() throws Exception {
  final SessionState ss = new SessionState(hiveConf);
  SessionState.start(ss);
  final Driver driver = new Driver(hiveConf);

  executeQuery(driver, "CREATE DATABASE " + db_general);
  createTbl(driver, db_general, g_student_user0, studentDef, studentData);
  createTbl(driver, db_general, g_voter_role0, voterDef, voterData);
  createTbl(driver, db_general, g_student_user2, studentDef, studentData);

  executeQuery(driver, "SET ROLE admin");
  executeQuery(driver, "CREATE ROLE " + test_role0);
  executeQuery(driver, "GRANT ROLE " + test_role0 + " TO USER " + org1Users[1]);
  executeQuery(driver, "GRANT ROLE " + test_role0 + " TO USER " + org1Users[2]);

  executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO USER %s", db_general, g_student_user0, org1Users[0]));
  executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO ROLE %s", db_general, g_voter_role0, test_role0));
  executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO USER %s", db_general, g_student_user2, org1Users[2]));
}
 

private ColumnarStruct readColumnarStruct(BytesRefArrayWritable buff, String schema) throws SerDeException {
     Pattern pcols = Pattern.compile("[a-zA-Z_0-9]*[ ]");
     List<String> types = HiveRCSchemaUtil.parseSchemaTypes(schema);
     List<String> cols = HiveRCSchemaUtil.parseSchema(pcols, schema);

     List<FieldSchema> fieldSchemaList = new ArrayList<FieldSchema>(
         cols.size());

     for (int i = 0; i < cols.size(); i++) {
         fieldSchemaList.add(new FieldSchema(cols.get(i), HiveRCSchemaUtil
             .findPigDataType(types.get(i))));
     }

     Properties props = new Properties();

     props.setProperty(Constants.LIST_COLUMNS,
         HiveRCSchemaUtil.listToString(cols));
     props.setProperty(Constants.LIST_COLUMN_TYPES,
         HiveRCSchemaUtil.listToString(types));

     Configuration hiveConf = new HiveConf(conf, SessionState.class);
     ColumnarSerDe serde = new ColumnarSerDe();
     serde.initialize(hiveConf, props);

     return (ColumnarStruct) serde.deserialize(buff);
}
 

public void execHiveSQLwithOverlay(final String sqlStmt,
    final String userName, Map<String, String> overLay) throws Exception {
  final HiveConf hiveConf = new HiveConf();
  for (Map.Entry<String, String> entry : overLay.entrySet()) {
    hiveConf.set(entry.getKey(), entry.getValue());
  }
  UserGroupInformation clientUgi = UserGroupInformation
      .createRemoteUser(userName);
  clientUgi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Void run() throws Exception {
      Driver driver = new Driver(hiveConf, userName);
      SessionState.start(new CliSessionState(hiveConf));
      CommandProcessorResponse cpr = driver.run(sqlStmt);
      if (cpr.getResponseCode() != 0) {
        throw new IOException("Failed to execute \"" + sqlStmt
            + "\". Driver returned " + cpr.getResponseCode() + " Error: "
            + cpr.getErrorMessage());
      }
      driver.close();
      SessionState.get().close();
      return null;
    }
  });
}
 

@Test(enabled = false)
public void testInsertIntoTempTable() throws Exception {
    String tableName = createTable();
    String insertTableName = createTable(false, false, true);
    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsNotRegistered(DEFAULT_DB, insertTableName, true);

    String query =
        "insert into " + insertTableName + " select id, name from " + tableName;

    runCommand(query);

    Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    Set<WriteEntity> outputs = getOutputs(insertTableName, Entity.Type.TABLE);
    outputs.iterator().next().setName(getQualifiedTblName(insertTableName + HiveMetaStoreBridge.TEMP_TABLE_PREFIX + SessionState.get().getSessionId()));
    outputs.iterator().next().setWriteType(WriteEntity.WriteType.INSERT);

    validateProcess(constructEvent(query,  HiveOperation.QUERY, inputs, outputs));

    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsRegistered(DEFAULT_DB, insertTableName, null, true);
}
 

private Task<? extends Serializable> analyzeGrantRevokeRole(boolean isGrant, ASTNode ast,
    HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException {
  List<PrincipalDesc> principalDesc = analyzePrincipalListDef(
      (ASTNode) ast.getChild(0));

  List<String> roles = new ArrayList<String>();
  for (int i = 1; i < ast.getChildCount(); i++) {
    roles.add(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(i).getText()));
  }
  String roleOwnerName = "";
  if (SessionState.get() != null
      && SessionState.get().getAuthenticator() != null) {
    roleOwnerName = SessionState.get().getAuthenticator().getUserName();
  }
  for (PrincipalDesc princ : principalDesc) {
    if (princ.getType() != PrincipalType.GROUP) {
      String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_ON_OBJECT + princ.getType();
      throw new SemanticException(msg);
    }
  }
  GrantRevokeRoleDDL grantRevokeRoleDDL = new GrantRevokeRoleDDL(isGrant,
      roles, principalDesc, roleOwnerName, PrincipalType.USER, false);
  return createTask(new DDLWork(inputs, outputs, grantRevokeRoleDDL));
}
 

public HiveAuthzBindingHook() throws Exception {
  SessionState session = SessionState.get();
  if(session == null) {
    throw new IllegalStateException("Session has not been started");
  }
  // HACK: set a random classname to force the Auth V2 in Hive
  SessionState.get().setAuthorizer(null);

  HiveConf hiveConf = session.getConf();
  if(hiveConf == null) {
    throw new IllegalStateException("Session HiveConf is null");
  }
  authzConf = loadAuthzConf(hiveConf);
  hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf);

  String serdeWhiteLists = authzConf.get(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST,
      HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST_DEFAULT);
  serdeWhiteList = Arrays.asList(serdeWhiteLists.split(","));
  serdeURIPrivilegesEnabled = authzConf.getBoolean(HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED,
      HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT);

  FunctionRegistry.setupPermissionsForBuiltinUDFs("", HiveAuthzConf.HIVE_UDF_BLACK_LIST);
}
 

private HiveConf newHiveConf() {
  HiveConf conf = new HiveConf(SessionState.class);

  MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "thrift://localhost:" + port);
  conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
  // Metastore needs to be set, and WITH the deprecated key :(
  // Otherwise, will default to /user/hive/warehouse when trying to create a new database
  // (database location is now sent by the client to the server...)
  HiveConf.setVar(conf, ConfVars.METASTOREWAREHOUSE, whDir);
  conf.set("mapred.job.tracker", "local");
  HiveConf.setVar(conf, ConfVars.SCRATCHDIR,  getTempDir("scratch_dir"));
  HiveConf.setVar(conf, ConfVars.LOCALSCRATCHDIR, getTempDir("local_scratch_dir"));
  HiveConf.setVar(conf, ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
  HiveConf.setBoolVar(conf, ConfVars.HIVE_CBO_ENABLED, false);

  return conf;

}
 

@VisibleForTesting
protected static AccessURI parseURI(String uri, boolean isLocal)
    throws SemanticException {
  try {
    HiveConf conf = SessionState.get().getConf();
    String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE);
    Path warehousePath = new Path(warehouseDir);
    if (warehousePath.isAbsoluteAndSchemeAuthorityNull()) {
      FileSystem fs = FileSystem.get(conf);
      warehouseDir = fs.makeQualified(warehousePath).toUri().toString();
    }
    return new AccessURI(PathUtils.parseURI(warehouseDir, uri, isLocal));
  } catch (Exception e) {
    throw new SemanticException("Error parsing URI " + uri + ": " +
      e.getMessage(), e);
  }
}
 

@Before
public void setup() throws Exception {
  conf = new HiveConf();
  baseDir = Files.createTempDir();
  baseDir.setWritable(true, false);
  conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath());
  SessionState.start(conf);
  conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
      SentryHiveAuthorizationTaskFactoryImpl.class.getName());

  db = Mockito.mock(Hive.class);
  table = new Table(DB, TABLE);
  partition = new Partition(table);
  context = new Context(conf);
  parseDriver = new ParseDriver();
  analyzer = new DDLSemanticAnalyzer(conf, db);
  SessionState.start(conf);
  Mockito.when(db.getTable(TABLE, false)).thenReturn(table);
  Mockito.when(db.getPartition(table, new HashMap<String, String>(), false))
  .thenReturn(partition);

  HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator();
  auth.setConf(conf);
  currentUser = auth.getUserName();

}
 

private static void generateHiveTestData() throws Exception {
  final SessionState ss = new SessionState(hiveConf);
  SessionState.start(ss);
  final Driver driver = new Driver(hiveConf);

  executeQuery(driver, "CREATE DATABASE " + db_general);
  createTbl(driver, db_general, g_student_user0, studentDef, studentData);
  createTbl(driver, db_general, g_voter_role0, voterDef, voterData);
  createTbl(driver, db_general, g_student_user2, studentDef, studentData);

  executeQuery(driver, "SET ROLE admin");
  executeQuery(driver, "CREATE ROLE " + test_role0);
  executeQuery(driver, "GRANT ROLE " + test_role0 + " TO USER " + org1Users[1]);
  executeQuery(driver, "GRANT ROLE " + test_role0 + " TO USER " + org1Users[2]);

  executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO USER %s", db_general, g_student_user0, org1Users[0]));
  executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO ROLE %s", db_general, g_voter_role0, test_role0));
  executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO USER %s", db_general, g_student_user2, org1Users[2]));
}
 

private void createHiveServer() {
  HiveServer2 server = new HiveServer2();
  server.init(new HiveConf());
  for (Service service : server.getServices()) {
    if (service instanceof CLIService) {
      _client = (CLIService) service;
    }
  }
  Preconditions.checkNotNull(_client, "CLI service not found in local Hive server");
  try {
    _sessionHandle = _client.openSession(null, null, null);
    _functionRegistry = SessionState.getRegistryForWrite();
    // "map_from_entries" UDF is required to create maps with non-primitive key types
    _functionRegistry.registerGenericUDF("map_from_entries", MapFromEntriesWrapper.class);
    // TODO: This is a hack. Hive's public API does not have a way to register an already created GenericUDF object
    // It only accepts a class name after which the parameterless constructor of the class is called to create a
    // GenericUDF object. This does not work for HiveTestStdUDFWrapper as it accepts the UDF classes as parameters.
    // However, Hive has an internal method which does allow passing GenericUDF objects instead of classes.
    _functionRegistryAddFunctionMethod =
        _functionRegistry.getClass().getDeclaredMethod("addFunction", String.class, FunctionInfo.class);
    _functionRegistryAddFunctionMethod.setAccessible(true);
  } catch (HiveSQLException | NoSuchMethodException e) {
    throw new RuntimeException(e);
  }
}
 

@BeforeClass
public void setUp() throws Exception {
    //  initialize 'driverWithNoHook' with HiveServer2 hook and HiveMetastore hook disabled
    HiveConf conf = new HiveConf();
    conf.set("hive.exec.post.hooks", "");
    conf.set("hive.metastore.event.listeners", "");

    SessionState ss = new SessionState(conf);
    ss = SessionState.start(ss);
    SessionState.setCurrentSessionState(ss);

    // Initialize 'driverWithNoHook'  with HS2 hook disabled and HMS hook disabled.
    driverWithNoHook = new Driver(conf);

    super.setUp();
}
 

public VariableSubstitution getVariableSubstitution() {
    // Make sure to set the session state for this thread before returning the VariableSubstitution. If not set,
    // hivevar:s will not be evaluated.
    SessionState.setCurrentSessionState(currentSessionState);

    SessionState ss = currentSessionState;
    return new VariableSubstitution(new HiveVariableSource() {
        @Override
        public Map<String, String> getHiveVariable() {
            return ss.getHiveVariables();
        }
    });
}
 

private static void deleteResource(SessionState value, ResourceType type) {
    // Hive < 0.14
    Method method = ReflectionUtils.findMethod(SessionState.class, "delete_resource", ResourceType.class);
    if (method == null) {
        method = ReflectionUtils.findMethod(SessionState.class, "delete_resources", ResourceType.class);
    }

    Assert.notNull(method, "Cannot detect delete resource(s) method on SessionState");
    ReflectionUtils.invoke(method, value, type);
}
 

/**
 * Filter the command "show columns in table"
 *
 */
private int showFilterColumns(ShowColumnsDesc showCols) throws HiveException {
  Table table = Hive.get(conf).getTable(showCols.getTableName());

  // write the results in the file
  DataOutputStream outStream = null;
  try {
    Path resFile = new Path(showCols.getResFile());
    FileSystem fs = resFile.getFileSystem(conf);
    outStream = fs.create(resFile);

    List<FieldSchema> cols = table.getCols();
    cols.addAll(table.getPartCols());
    // In case the query is served by HiveServer2, don't pad it with spaces,
    // as HiveServer2 output is consumed by JDBC/ODBC clients.
    boolean isOutputPadded = !SessionState.get().isHiveServerQuery();
    outStream.writeBytes(MetaDataFormatUtils.getAllColumnsInformation(
        fiterColumns(cols, table), false, isOutputPadded, null));
    outStream.close();
    outStream = null;
  } catch (IOException e) {
    throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
  } finally {
    IOUtils.closeStream(outStream);
  }
  return 0;
}
 

/**
 * Will start the HiveServer.
 *
 * @param testConfig Specific test case properties. Will be merged with the HiveConf of the context
 * @param hiveVars   HiveVars to pass on to the HiveServer for this session
 */
public void init(Map<String, String> testConfig, Map<String, String> hiveVars) {

    context.init();

    HiveConf hiveConf = context.getHiveConf();

    // merge test case properties with hive conf before HiveServer is started.
    for (Map.Entry<String, String> property : testConfig.entrySet()) {
        hiveConf.set(property.getKey(), property.getValue());
    }

    try {
        hiveServer2 = new HiveServer2();
        hiveServer2.init(hiveConf);

        // Locate the ClIService in the HiveServer2
        for (Service service : hiveServer2.getServices()) {
            if (service instanceof CLIService) {
                client = (CLIService) service;
            }
        }

        Preconditions.checkNotNull(client, "ClIService was not initialized by HiveServer2");

        sessionHandle = client.openSession("noUser", "noPassword", null);

        SessionState sessionState = client.getSessionManager().getSession(sessionHandle).getSessionState();
        currentSessionState = sessionState;
        currentSessionState.setHiveVariables(hiveVars);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to create HiveServer :" + e.getMessage(), e);
    }

    // Ping hive server before we do anything more with it! If validation
    // is switched on, this will fail if metastorage is not set up properly
    pingHiveServer();
}
 

private static DatabaseTable parseDBTable(String obj) throws HiveException {
  String[] dbTab = Iterables.toArray(DB_TBL_SPLITTER.split(obj), String.class);
  if (dbTab.length == 2) {
    return new DatabaseTable(dbTab[0], dbTab[1]);
  } else if (dbTab.length == 1){
    return new DatabaseTable(SessionState.get().getCurrentDatabase(), obj);
  } else {
    String msg = "Malformed database.table '" + obj + "'";
    throw new HiveException(msg);
  }
}
 

private void extractDbAndTb(String tableName) {
  if (tableName.contains(".")) {
    String[] tb = tableName.split("\\.");
    currentDb = tb[0];
    currentTb = tb[1];
  } else {
    currentDb = SessionState.get().getCurrentDatabase();
    currentTb = tableName;
  }
}
 

@Override
public Task<? extends Serializable> createGrantTask(ASTNode ast, HashSet<ReadEntity> inputs,
    HashSet<WriteEntity> outputs) throws SemanticException {
  List<PrivilegeDesc> privilegeDesc = analyzePrivilegeListDef(
      (ASTNode) ast.getChild(0));
  List<PrincipalDesc> principalDesc = analyzePrincipalListDef(
      (ASTNode) ast.getChild(1));
  SentryHivePrivilegeObjectDesc privilegeObj = null;
  boolean grantOption = false;
  if (ast.getChildCount() > 2) {
    for (int i = 2; i < ast.getChildCount(); i++) {
      ASTNode astChild = (ASTNode) ast.getChild(i);
      if (astChild.getType() == HiveParser.TOK_GRANT_WITH_OPTION) {
        grantOption = true;
      } else if (astChild.getType() == HiveParser.TOK_PRIV_OBJECT) {
        privilegeObj = analyzePrivilegeObject(astChild);
      }
    }
  }
  String userName = null;
  if (SessionState.get() != null
      && SessionState.get().getAuthenticator() != null) {
    userName = SessionState.get().getAuthenticator().getUserName();
  }
  Preconditions.checkNotNull(privilegeObj, "privilegeObj is null for " + ast.dump());
  if (privilegeObj.getPartSpec() != null) {
    throw new SemanticException(SentryHiveConstants.PARTITION_PRIVS_NOT_SUPPORTED);
  }
  for (PrincipalDesc princ : principalDesc) {
    if (princ.getType() != PrincipalType.ROLE) {
      String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_FOR_PRINCIPAL + princ.getType();
      throw new SemanticException(msg);
    }
  }
  GrantDesc grantDesc = new GrantDesc(privilegeObj, privilegeDesc,
      principalDesc, userName, PrincipalType.USER, grantOption);
  return createTask(new DDLWork(inputs, outputs, grantDesc));
}
 

private HiveOperation getCurrentHiveStmtOp() {
  SessionState sessState = SessionState.get();
  if (sessState == null) {
    LOG.warn("SessionState is null");
    return null;
  }
  return sessState.getHiveOperation();
}
 

public HiveAuthzBindingHookV2() throws Exception {
  SessionState session = SessionState.get();
  if(session == null) {
    throw new IllegalStateException("Session has not been started");
  }

  HiveConf hiveConf = session.getConf();
  if(hiveConf == null) {
    throw new IllegalStateException("Session HiveConf is null");
  }
  authzConf = HiveAuthzBindingHook.loadAuthzConf(hiveConf);
  hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf);
}
 

private void executeOnFailureHooks(HiveOperation hiveOp, SentryAccessDeniedException e)
    throws HiveAccessControlException {
  SentryOnFailureHookContext hookCtx =
      new SentryOnFailureHookContextImpl(SessionState.get().getCmd(), null, null, hiveOp, null,
          null, null, null, authenticator.getUserName(), null, new AuthorizationException(e),
          authzConf);
  SentryAuthorizerUtil.executeOnFailureHooks(hookCtx, authzConf);
  throw new HiveAccessControlException(e.getMessage(), e);
}
 

private HiveOperation getCurrentHiveStmtOp() {
  SessionState sessState = SessionState.get();
  if (sessState == null) {
    // TODO: Warn
    return null;
  }
  return sessState.getHiveOperation();
}
 

@Test public void testCurrentDate() {
  executeSql("select current_date");
  Assert.assertEquals(
      SqlFunctions.toInt(java.sql.Date.valueOf(
          SessionState.get()
              .getQueryCurrentTimestamp()
              .toString()
              .substring(0, 10)), TimeZone.getTimeZone("UTC")),
      executeSql("select current_date as c1").get(0)
          .get("c1"));
}
 

private boolean filterWriteEntity(WriteEntity writeEntity)
    throws AuthorizationException {
  // skip URI validation for session scratch file URIs
  if (writeEntity.isTempURI()) {
    return true;
  }
  try {
    if (writeEntity.getTyp().equals(Type.DFS_DIR)
        || writeEntity.getTyp().equals(Type.LOCAL_DIR)) {
      HiveConf conf = SessionState.get().getConf();
      String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE);
      URI scratchURI = new URI(PathUtils.parseDFSURI(warehouseDir,
        conf.getVar(HiveConf.ConfVars.SCRATCHDIR)));
      URI requestURI = new URI(PathUtils.parseDFSURI(warehouseDir,
        writeEntity.getLocation().getPath()));
      LOG.debug("scratchURI = " + scratchURI + ", requestURI = " + requestURI);
      if (PathUtils.impliesURI(scratchURI, requestURI)) {
        return true;
      }
      URI localScratchURI = new URI(PathUtils.parseLocalURI(conf.getVar(HiveConf.ConfVars.LOCALSCRATCHDIR)));
      URI localRequestURI = new URI(PathUtils.parseLocalURI(writeEntity.getLocation().getPath()));
      LOG.debug("localScratchURI = " + localScratchURI + ", localRequestURI = " + localRequestURI);
      if (PathUtils.impliesURI(localScratchURI, localRequestURI)) {
        return true;
      }
    }
  } catch (Exception e) {
    throw new AuthorizationException("Failed to extract uri details", e);
  }
  return false;
}
 

@BeforeClass
public static void setupTestURI() {
  conf = new HiveConf();
  baseDir = Files.createTempDir();
  baseDir.setWritable(true, false);
  conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath());
  SessionState.start(conf);
}
 

@Override protected void executeBeforeSqlQuery(String sql) {
  //clear thread context
  HiveUDFInstanceCollecterPerSqlQuery.clear();
  //prepare hive session state per sql query
  if (SessionState.get() == null) {
    SessionState ss = new SessionState(HIVE_CONF);
    SessionState.setCurrentSessionState(ss);
  }
  SessionState.get().setupQueryCurrentTimestamp();
}
 

private HiveConf newHiveConf() {
  HiveConf conf = new HiveConf(SessionState.class);

  HiveConf.setVar(conf, ConfVars.METASTOREURIS, "thrift://localhost:" + port);
  conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
  HiveConf.setVar(conf, ConfVars.METASTOREWAREHOUSE, whDir);
  conf.set("mapred.job.tracker", "local");
  HiveConf.setVar(conf, ConfVars.SCRATCHDIR, getTempDir("scratch_dir"));
  HiveConf.setVar(conf, ConfVars.LOCALSCRATCHDIR, getTempDir("local_scratch_dir"));
  HiveConf.setVar(conf, ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
  HiveConf.setBoolVar(conf, ConfVars.HIVE_CBO_ENABLED, false);
  return conf;

}
 

/**
    * Does the configuration setup and schema parsing and setup.
    *
    * @param table_schema
    *            String
    * @param columnsToRead
    *            String
    */
   private void setup(String table_schema) {

if (table_schema == null)
    throw new RuntimeException(
	    "The table schema must be defined as colname type, colname type.  All types are hive types");

// create basic configuration for hdfs and hive
conf = new Configuration();
hiveConf = new HiveConf(conf, SessionState.class);

// parse the table_schema string
List<String> types = HiveRCSchemaUtil.parseSchemaTypes(table_schema);
List<String> cols = HiveRCSchemaUtil.parseSchema(pcols, table_schema);

List<FieldSchema> fieldSchemaList = new ArrayList<FieldSchema>(
	cols.size());

for (int i = 0; i < cols.size(); i++) {
    fieldSchemaList.add(new FieldSchema(cols.get(i), HiveRCSchemaUtil
	    .findPigDataType(types.get(i))));
}

pigSchema = new ResourceSchema(new Schema(fieldSchemaList));

props = new Properties();

// setting table schema properties for ColumnarSerDe
// these properties are never changed by the columns to read filter,
// because the columnar serde needs to now the
// complete format of each record.
props.setProperty(Constants.LIST_COLUMNS,
	HiveRCSchemaUtil.listToString(cols));
props.setProperty(Constants.LIST_COLUMN_TYPES,
	HiveRCSchemaUtil.listToString(types));

   }