Java Code Examples for org.apache.tez.common.TezUtils#createByteStringFromConf()

The following examples show how to use org.apache.tez.common.TezUtils#createByteStringFromConf() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MRHelpers.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@LimitedPrivate("Hive, Pig")
public static ByteString createByteStringFromConf(Configuration conf)
    throws IOException {
  return TezUtils.createByteStringFromConf(conf);
}
Example 2
Source File: TestMRInputSplitDistributor.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void testSerializedPayload() throws IOException {

  Configuration conf = new Configuration(false);
  conf.setBoolean(MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD, true);
  ByteString confByteString = TezUtils.createByteStringFromConf(conf);
  InputSplit split1 = new InputSplitForTest(1);
  InputSplit split2 = new InputSplitForTest(2);
  MRSplitProto proto1 = MRInputHelpers.createSplitProto(split1);
  MRSplitProto proto2 = MRInputHelpers.createSplitProto(split2);
  MRSplitsProto.Builder splitsProtoBuilder = MRSplitsProto.newBuilder();
  splitsProtoBuilder.addSplits(proto1);
  splitsProtoBuilder.addSplits(proto2);
  MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
  payloadProto.setSplits(splitsProtoBuilder.build());
  payloadProto.setConfigurationBytes(confByteString);
  UserPayload userPayload =
      UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());

  InputInitializerContext context = new TezTestUtils.TezRootInputInitializerContextForTest(userPayload,
      new Configuration(false));
  MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);

  List<Event> events = splitDist.initialize();

  assertEquals(3, events.size());
  assertTrue(events.get(0) instanceof InputUpdatePayloadEvent);
  assertTrue(events.get(1) instanceof InputDataInformationEvent);
  assertTrue(events.get(2) instanceof InputDataInformationEvent);

  InputDataInformationEvent diEvent1 = (InputDataInformationEvent) (events.get(1));
  InputDataInformationEvent diEvent2 = (InputDataInformationEvent) (events.get(2));

  assertNull(diEvent1.getDeserializedUserPayload());
  assertNull(diEvent2.getDeserializedUserPayload());

  assertNotNull(diEvent1.getUserPayload());
  assertNotNull(diEvent2.getUserPayload());

  MRSplitProto event1Proto = MRSplitProto.parseFrom(ByteString.copyFrom(diEvent1.getUserPayload()));
  InputSplit is1 = MRInputUtils.getOldSplitDetailsFromEvent(event1Proto, new Configuration());
  assertTrue(is1 instanceof InputSplitForTest);
  assertEquals(1, ((InputSplitForTest) is1).identifier);

  MRSplitProto event2Proto = MRSplitProto.parseFrom(ByteString.copyFrom(diEvent2.getUserPayload()));
  InputSplit is2 = MRInputUtils.getOldSplitDetailsFromEvent(event2Proto, new Configuration());
  assertTrue(is2 instanceof InputSplitForTest);
  assertEquals(2, ((InputSplitForTest) is2).identifier);
}
Example 3
Source File: TestMRInputSplitDistributor.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void testDeserializedPayload() throws IOException {

  Configuration conf = new Configuration(false);
  conf.setBoolean(MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD, false);
  ByteString confByteString = TezUtils.createByteStringFromConf(conf);
  InputSplit split1 = new InputSplitForTest(1);
  InputSplit split2 = new InputSplitForTest(2);
  MRSplitProto proto1 = MRInputHelpers.createSplitProto(split1);
  MRSplitProto proto2 = MRInputHelpers.createSplitProto(split2);
  MRSplitsProto.Builder splitsProtoBuilder = MRSplitsProto.newBuilder();
  splitsProtoBuilder.addSplits(proto1);
  splitsProtoBuilder.addSplits(proto2);
  MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
  payloadProto.setSplits(splitsProtoBuilder.build());
  payloadProto.setConfigurationBytes(confByteString);
  UserPayload userPayload =
      UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());

  InputInitializerContext context = new TezTestUtils.TezRootInputInitializerContextForTest(userPayload,
      new Configuration(false));
  MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);

  List<Event> events = splitDist.initialize();

  assertEquals(3, events.size());
  assertTrue(events.get(0) instanceof InputUpdatePayloadEvent);
  assertTrue(events.get(1) instanceof InputDataInformationEvent);
  assertTrue(events.get(2) instanceof InputDataInformationEvent);

  InputDataInformationEvent diEvent1 = (InputDataInformationEvent) (events.get(1));
  InputDataInformationEvent diEvent2 = (InputDataInformationEvent) (events.get(2));

  assertNull(diEvent1.getUserPayload());
  assertNull(diEvent2.getUserPayload());

  assertNotNull(diEvent1.getDeserializedUserPayload());
  assertNotNull(diEvent2.getDeserializedUserPayload());

  assertTrue(diEvent1.getDeserializedUserPayload() instanceof InputSplitForTest);
  assertEquals(1, ((InputSplitForTest) diEvent1.getDeserializedUserPayload()).identifier);

  assertTrue(diEvent2.getDeserializedUserPayload() instanceof InputSplitForTest);
  assertEquals(2, ((InputSplitForTest) diEvent2.getDeserializedUserPayload()).identifier);
}
Example 4
Source File: GraceShuffleVertexManagerForTest.java    From tez with Apache License 2.0 4 votes vote down vote up 
public ByteString toByteString() throws IOException {
  return TezUtils.createByteStringFromConf(build().toConfiguration());
}