Java Code Examples for org.apache.hadoop.mapred.JobConf#addResource()

The following examples show how to use org.apache.hadoop.mapred.JobConf#addResource() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: InputFormatTestUtil.java    From hudi with Apache License 2.0 6 votes vote down vote up 
public static void setPropsForInputFormat(JobConf jobConf,
    Schema schema, String hiveColumnTypes) {
  List<Schema.Field> fields = schema.getFields();
  String names = fields.stream().map(f -> f.name().toString()).collect(Collectors.joining(","));
  String postions = fields.stream().map(f -> String.valueOf(f.pos())).collect(Collectors.joining(","));
  Configuration conf = HoodieTestUtils.getDefaultHadoopConf();

  String hiveColumnNames = fields.stream().filter(field -> !field.name().equalsIgnoreCase("datestr"))
      .map(Schema.Field::name).collect(Collectors.joining(","));
  hiveColumnNames = hiveColumnNames + ",datestr";
  String modifiedHiveColumnTypes = HoodieAvroUtils.addMetadataColumnTypes(hiveColumnTypes);
  modifiedHiveColumnTypes = modifiedHiveColumnTypes + ",string";
  jobConf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
  jobConf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, modifiedHiveColumnTypes);
  jobConf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, names);
  jobConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, postions);
  jobConf.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "datestr");
  conf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
  conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, names);
  conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, postions);
  conf.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "datestr");
  conf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, modifiedHiveColumnTypes);
  jobConf.addResource(conf);
}
Example 2
Source File: PostExPerformanceDiagnoser.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * read and populate job statistics information.
 */
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {

  /*
   * Convert the input strings to URL
   */
  URL jobConfFileUrl = new URL(this._jobConfFile);
  URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
  
  /*
   * Read the Job Configuration from the jobConfFile url
   */  
  jobConf.addResource(jobConfFileUrl);
  
  /* 
   * Read JobHistoryFile and build job counters to evaluate diagnostic rules
   */
  if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
  } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
  } else {
    throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
  }
}
Example 3
Source File: PostExPerformanceDiagnoser.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
/**
 * read and populate job statistics information.
 */
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {

  /*
   * Convert the input strings to URL
   */
  URL jobConfFileUrl = new URL(this._jobConfFile);
  URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
  
  /*
   * Read the Job Configuration from the jobConfFile url
   */  
  jobConf.addResource(jobConfFileUrl);
  
  /* 
   * Read JobHistoryFile and build job counters to evaluate diagnostic rules
   */
  if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
  } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
  } else {
    throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
  }
}
Example 4
Source File: HoodieMergeOnReadTestUtils.java    From hudi with Apache License 2.0 5 votes vote down vote up 
private static void setPropsForInputFormat(FileInputFormat inputFormat, JobConf jobConf, Schema schema,
                                           String basePath) {
  List<Schema.Field> fields = schema.getFields();
  String names = fields.stream().map(f -> f.name().toString()).collect(Collectors.joining(","));
  String postions = fields.stream().map(f -> String.valueOf(f.pos())).collect(Collectors.joining(","));
  Configuration conf = HoodieTestUtils.getDefaultHadoopConf();

  String hiveColumnNames = fields.stream().filter(field -> !field.name().equalsIgnoreCase("datestr"))
      .map(Schema.Field::name).collect(Collectors.joining(","));
  hiveColumnNames = hiveColumnNames + ",datestr";

  String hiveColumnTypes = HoodieAvroUtils.addMetadataColumnTypes(HoodieTestDataGenerator.TRIP_HIVE_COLUMN_TYPES);
  hiveColumnTypes = hiveColumnTypes + ",string";
  jobConf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
  jobConf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, hiveColumnTypes);
  jobConf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, names);
  jobConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, postions);
  jobConf.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "datestr");
  conf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
  conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, names);
  conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, postions);
  conf.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "datestr");
  conf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, hiveColumnTypes);

  // Hoodie Input formats are also configurable
  Configurable configurable = (Configurable)inputFormat;
  configurable.setConf(conf);
  jobConf.addResource(conf);
}
Example 5
Source File: MiniDFS.java    From vxquery with Apache License 2.0 5 votes vote down vote up 
public void startHDFS(String folder) throws IOException {

        JobConf conf = new JobConf();
        conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
        conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
        conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
        int numDataNodes = 1;
        int nameNodePort = 40000;

        System.setProperty("hadoop.log.dir", "logs");
        System.setProperty("test.build.data", folder.concat("/"));
        MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf);
        build.nameNodePort(nameNodePort);
        build.nameNodeHttpPort(nameNodePort + 34);
        build.numDataNodes(numDataNodes);
        build.checkExitOnShutdown(true);
        build.startupOption(StartupOption.REGULAR);
        build.format(true);
        build.waitSafeMode(true);
        dfsCluster = build.build();

        FileSystem dfs = FileSystem.get(conf);
        Path src = new Path(DATA_PATH);
        dfs.mkdirs(new Path("/tmp"));
        Path dest = new Path("/tmp/vxquery-hdfs-test");
        dfs.copyFromLocalFile(src, dest);
        if (dfs.exists(dest)) {
            System.err.println("Test files copied to HDFS successfully");
        }
        dfs.close();
    }
Example 6
Source File: TestTableOutputFormatConnectionExhaust.java    From hbase with Apache License 2.0 5 votes vote down vote up 
/**
 * Open and close a TableOutputFormat.  The closing the RecordWriter should release HBase
 * Connection (ZK) resources, and will throw exception if they are exhausted.
 */
static void openCloseTableOutputFormat(int iter)  throws IOException {
  LOG.info("Instantiating TableOutputFormat connection  " + iter);
  JobConf conf = new JobConf();
  conf.addResource(UTIL.getConfiguration());
  conf.set(TableOutputFormat.OUTPUT_TABLE, TABLE);
  TableMapReduceUtil.initTableMapJob(TABLE, FAMILY, TableMap.class,
      ImmutableBytesWritable.class, ImmutableBytesWritable.class, conf);
  TableOutputFormat tof = new TableOutputFormat();
  RecordWriter rw = tof.getRecordWriter(null, conf, TABLE, null);
  rw.close(null);
}
Example 7
Source File: HExecutionEngine.java    From spork with Apache License 2.0 5 votes vote down vote up 
public JobConf getS3Conf() throws ExecException {
    JobConf jc = new JobConf();
    jc.addResource(CORE_SITE);
    Iterator<Entry<String, String>> i = jc.iterator();
    while (i.hasNext()) {
        Entry<String, String> e = i.next();
        String key = e.getKey();
        String value = e.getValue();
        if (key.startsWith("fs.s3") || key.startsWith("fs.s3n")) {
            jc.set(key, value);
        }
    }
    return jc;
}
Example 8
Source File: HExecutionEngine.java    From spork with Apache License 2.0 5 votes vote down vote up 
public JobConf getLocalConf() {
    JobConf jc = new JobConf(false);

    jc.addResource(CORE_DEFAULT_SITE);
    jc.addResource(MAPRED_DEFAULT_SITE);
    jc.addResource(YARN_DEFAULT_SITE);

    return jc;
}
Example 9
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
    
    validateInputParam(containerIdStr,
        Environment.CONTAINER_ID.name());
    validateInputParam(nodeHostString, Environment.NM_HOST.name());
    validateInputParam(nodePortString, Environment.NM_PORT.name());
    validateInputParam(nodeHttpPortString,
        Environment.NM_HTTP_PORT.name());
    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();
    long appSubmitTime = Long.parseLong(appSubmitTimeStr);
    
    
    MRAppMaster appMaster =
        new MRAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), appSubmitTime);
    ShutdownHookManager.get().addShutdownHook(
      new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
    JobConf conf = new JobConf(new YarnConfiguration());
    conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
    
    MRWebAppUtil.initialize(conf);
    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());
    conf.set(MRJobConfig.USER_NAME, jobUserName);
    initAndStartAppMaster(appMaster, conf, jobUserName);
  } catch (Throwable t) {
    LOG.fatal("Error starting MRAppMaster", t);
    ExitUtil.terminate(1, t);
  }
}
Example 10
Source File: MRAppMaster.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
    
    validateInputParam(containerIdStr,
        Environment.CONTAINER_ID.name());
    validateInputParam(nodeHostString, Environment.NM_HOST.name());
    validateInputParam(nodePortString, Environment.NM_PORT.name());
    validateInputParam(nodeHttpPortString,
        Environment.NM_HTTP_PORT.name());
    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();
    long appSubmitTime = Long.parseLong(appSubmitTimeStr);
    
    
    MRAppMaster appMaster =
        new MRAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), appSubmitTime);
    ShutdownHookManager.get().addShutdownHook(
      new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
    JobConf conf = new JobConf(new YarnConfiguration());
    conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
    
    MRWebAppUtil.initialize(conf);
    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());
    conf.set(MRJobConfig.USER_NAME, jobUserName);
    initAndStartAppMaster(appMaster, conf, jobUserName);
  } catch (Throwable t) {
    LOG.fatal("Error starting MRAppMaster", t);
    ExitUtil.terminate(1, t);
  }
}