Java Code Examples for org.apache.tez.common.TezUtils#createConfFromByteString()

The following examples show how to use org.apache.tez.common.TezUtils#createConfFromByteString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestMRRJobsDAGApi.java From tez with Apache License 2.0

6 votes

@Override
public List<Event> initialize()  throws Exception {
  MRInputUserPayloadProto userPayloadProto = MRInputHelpers
      .parseMRInputPayload(getContext().getInputUserPayload());
  Configuration conf = TezUtils.createConfFromByteString(userPayloadProto
      .getConfigurationBytes());

  try {
    Thread.currentThread().setContextClassLoader(TezClassLoader.getInstance());
    ReflectionUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME);
    LOG.info("Class found");
    FileSystem fs = FileSystem.get(conf);
    fs.mkdirs(new Path("/tmp/relocalizationfilefound"));
  } catch (TezReflectionException e) {
    LOG.info("Class not found");
  }

  return super.initialize();
}

Example 2

Source File: MRHelpers.java From incubator-tez with Apache License 2.0

4 votes

@LimitedPrivate("Hive, Pig")
public static Configuration createConfFromByteString(ByteString bs)
    throws IOException {
  return TezUtils.createConfFromByteString(bs);
}

Example 3

Source File: MRInputSplitDistributor.java From tez with Apache License 2.0

4 votes

@Override
public List<Event> initialize() throws IOException {
  StopWatch sw = new StopWatch().start();
  MRInputUserPayloadProto userPayloadProto = MRInputHelpers
      .parseMRInputPayload(getContext().getInputUserPayload());
  sw.stop();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Time to parse MRInput payload into prot: "
        + sw.now(TimeUnit.MILLISECONDS));
  }
  Configuration conf = TezUtils.createConfFromByteString(userPayloadProto
      .getConfigurationBytes());
  JobConf jobConf = new JobConf(conf);
  boolean useNewApi = jobConf.getUseNewMapper();
  sendSerializedEvents = conf.getBoolean(
      MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD,
      MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD_DEFAULT);
  LOG.info("Emitting serialized splits: " + sendSerializedEvents);

  this.splitsProto = userPayloadProto.getSplits();
  
  MRInputUserPayloadProto.Builder updatedPayloadBuilder = MRInputUserPayloadProto.newBuilder(userPayloadProto);
  updatedPayloadBuilder.clearSplits();

  List<Event> events = Lists.newArrayListWithCapacity(this.splitsProto.getSplitsCount() + 1);
  InputUpdatePayloadEvent updatePayloadEvent = InputUpdatePayloadEvent.create(
      updatedPayloadBuilder.build().toByteString().asReadOnlyByteBuffer());

  events.add(updatePayloadEvent);
  int count = 0;

  for (MRSplitProto mrSplit : this.splitsProto.getSplitsList()) {

    InputDataInformationEvent diEvent;

    if (sendSerializedEvents) {
      // Unnecessary array copy, can be avoided by using ByteBuffer instead of
      // a raw array.
      diEvent = InputDataInformationEvent.createWithSerializedPayload(count++,
          mrSplit.toByteString().asReadOnlyByteBuffer());
    } else {
      if (useNewApi) {
        org.apache.hadoop.mapreduce.InputSplit newInputSplit = MRInputUtils
            .getNewSplitDetailsFromEvent(mrSplit, conf);
        diEvent = InputDataInformationEvent.createWithObjectPayload(count++, newInputSplit);
      } else {
        org.apache.hadoop.mapred.InputSplit oldInputSplit = MRInputUtils
            .getOldSplitDetailsFromEvent(mrSplit, conf);
        diEvent = InputDataInformationEvent.createWithObjectPayload(count++, oldInputSplit);
      }
    }
    events.add(diEvent);
  }

  return events;
}