org.apache.tez.dag.api.Vertex Java Examples

private DAG createDag(TezConfiguration tezConf, Path outPath, long outSize, int extraColumns, int numTasks)
        throws IOException {

    long largeOutSizePerTask = outSize / numTasks;

    DAG dag = DAG.create("TopK DataGen");

    Vertex genDataVertex = Vertex.create("datagen", ProcessorDescriptor.create(
                    GenDataProcessor.class.getName()).setUserPayload(
                    UserPayload.create(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask, extraColumns)))),
            numTasks);
    genDataVertex.addDataSink(OUTPUT,
            MROutput.createConfigBuilder(new Configuration(tezConf),
                    TextOutputFormat.class, outPath.toUri().toString()).build());
    dag.addVertex(genDataVertex);

    return dag;
}
 

@Ignore
@Test (timeout = 60000)
public void testTaskEventsProcessingSpeed() throws Exception {
  Logger.getRootLogger().setLevel(Level.WARN);
  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  tezconf.setBoolean(TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER, true);
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null,
      null, false, false, 30, 1000);
  tezClient.start();

  final String vAName = "A";
  
  DAG dag = DAG.create("testTaskEventsProcessingSpeed");
  Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 50000);
  dag.addVertex(vA);

  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  mockApp.doSleep = false;
  DAGClient dagClient = tezClient.submitDAG(dag);
  DAGStatus status = dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
  tezClient.stop();
}
 

private GroupInputEdge newGroupInputEdge(TezOperator fromOp,
        TezOperator toOp, VertexGroup from, Vertex to) throws IOException {

    EdgeProperty edgeProperty = newEdge(fromOp, toOp);

    String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName();

    // In case of SCATTER_GATHER and UnorderedKVInput it will still be
    // ConcatenatedMergedKeyValueInput
    if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER)
            && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) {
        groupInputClass = OrderedGroupedMergedKVInput.class.getName();
    }

    return GroupInputEdge.create(from, to, edgeProperty,
            InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())
                .setHistoryText(edgeProperty.getEdgeDestination().getHistoryText()));
}
 

@Test(timeout = 5000)
public void testSubmitDAGAppFailed() throws Exception {
  final TezClientForTest client = configureAndCreateTezClient();
  client.start();
  
  client.callRealGetSessionAMProxy = true;
  String msg = "Application Test Failed";
  when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
      .thenReturn(YarnApplicationState.KILLED);
  when(client.mockYarnClient.getApplicationReport(client.mockAppId).getDiagnostics()).thenReturn(
      msg);

  Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1,
      Resource.newInstance(1, 1));
  DAG dag = DAG.create("DAG").addVertex(vertex);
  
  try {
    client.submitDAG(dag);
    fail();
  } catch (SessionNotRunning e) {
    assertTrue(e.getMessage().contains(msg));
  }
  client.stop();
}
 

@Test(timeout = 20000)
public void testSelfKillReported() throws IOException, TezException, InterruptedException {

  TezClient tezClient = getTezClient("testSelfKillReported");
  DAGClient dagClient = null;

  try {
    FailingProcessor.configureForKilled(10);
    DAG dag = DAG.create("testSelfKillReported").addVertex(
        Vertex
            .create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));

    dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    assertEquals(10, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getKilledTaskAttemptCount());
  } finally {
    if (dagClient != null) {
      dagClient.close();
    }
    tezClient.stop();
  }
}
 

@Test(timeout = 20000)
public void testFatalErrorReported() throws IOException, TezException, InterruptedException {

  TezClient tezClient = getTezClient("testFatalErrorReported");
  DAGClient dagClient = null;

  try {
    FailingProcessor.configureForFatalFail();
    DAG dag = DAG.create("testFatalErrorReportedDag").addVertex(
        Vertex
            .create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));

    dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
    assertEquals(1, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount());
  } finally {
    if (dagClient != null) {
      dagClient.close();
    }
    tezClient.stop();
  }
}
 

@Override
public void checkGroupNonConstWithParallelResult(PhysicalPlan pp, PigContext pc) throws Exception {
    TezOperPlan tezPlan = buildTezPlan(pp, pc);

    LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
    loaderStorer.visit();

    ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
    parallelismSetter.visit();

    DAG tezDag = getTezDAG(tezPlan, pc);
    TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
    dagBuilder.visit();
    for (Vertex v : tezDag.getVertices()) {
        if (!v.getInputVertices().isEmpty()) {
            assertEquals(v.getParallelism(), 100);
        }
    }
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 

/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  return dag;
}
 

private void runJoinValidate(TezClient tezClient, String name, int extExpectedCount,
                             Vertex.VertexExecutionContext lhsContext,
                             Vertex.VertexExecutionContext rhsContext,
                             Vertex.VertexExecutionContext validateContext) throws
    Exception {
  int externalSubmissionCount = tezTestServiceCluster.getNumSubmissions();

  TezConfiguration tezConf = new TezConfiguration(confForJobs);
  JoinValidateConfigured joinValidate =
      new JoinValidateConfigured(null, lhsContext, rhsContext,
          validateContext, name);
  String[] validateArgs = new String[]{"-disableSplitGrouping",
      HASH_JOIN_EXPECTED_RESULT_PATH.toString(), HASH_JOIN_OUTPUT_PATH.toString(), "3"};
  assertEquals(0, joinValidate.run(tezConf, validateArgs, tezClient));

  // Ensure this was actually submitted to the external cluster
  assertEquals(extExpectedCount,
      (tezTestServiceCluster.getNumSubmissions() - externalSubmissionCount));
}
 

@Test (timeout = 10000)
public void testSchedulerErrorHandling() throws Exception {
  TezConfiguration tezconf = new TezConfiguration(defaultConf);

  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();

  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(false);

  DAG dag = DAG.create("testSchedulerErrorHandling");
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
  dag.addVertex(vA);

  tezClient.submitDAG(dag);
  mockLauncher.waitTillContainersLaunched();
  mockApp.handle(new DAGAppMasterEventSchedulingServiceError(
      org.apache.hadoop.util.StringUtils.stringifyException(new RuntimeException("Mock error"))));

  while(!mockApp.getShutdownHandler().wasShutdownInvoked()) {
    Thread.sleep(100);
  }
  Assert.assertEquals(DAGState.RUNNING, mockApp.getContext().getCurrentDAG().getState());
}
 

@Ignore
@Test (timeout = 600000)
public void testMemoryOneToOne() throws Exception {
  DAG dag = DAG.create("testMemoryOneToOne");
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks);
  Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), numTasks);
  dag.addVertex(vA)
      .addVertex(vB)
      .addEdge(
          Edge.create(vA, vB, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
              DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
              OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
  testMemory(dag, true);
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addEdge(Edge.create(v1, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

@Ignore
@Test (timeout = 600000)
public void testMemoryScatterGather() throws Exception {
  DAG dag = DAG.create("testMemoryScatterGather");
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks);
  Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), numTasks);
  dag.addVertex(vA)
      .addVertex(vB)
      .addEdge(
          Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER,
              DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
              OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
  testMemory(dag, true);
}
 

@Test(timeout = 5000)
public void testDAGFinishedRecoveryError() throws Exception {
  TezConfiguration tezconf = new TezConfiguration(defaultConf);

  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();

  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  mockApp.recoveryFatalError = true;
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(true);

  DAG dag = DAG.create("test");
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
  dag.addVertex(vA);

  DAGClient dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  while(!mockApp.getShutdownHandler().wasShutdownInvoked()) {
    Thread.sleep(100);
  }
  Assert.assertEquals(DAGState.SUCCEEDED, mockApp.getContext().getCurrentDAG().getState());
  Assert.assertEquals(DAGAppMasterState.FAILED, mockApp.getState());
  Assert.assertTrue(StringUtils.join(mockApp.getDiagnostics(),",")
      .contains("Recovery had a fatal error, shutting down session after" +
            " DAG completion"));
}
 

@Test(timeout = 60000)
public void testVertexFailuresMaxPercent() throws TezException, InterruptedException, IOException {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, "50.0f");
  tezConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
  TezClient tezClient = TezClient.create("TestVertexFailuresMaxPercent", tezConf);
  tezClient.start();

  try {
    DAG dag = DAG.create("TestVertexFailuresMaxPercent");
    Vertex vertex1 = Vertex.create("Parent", ProcessorDescriptor.create(
        FailingAttemptProcessor.class.getName()), 2);
    Vertex vertex2 = Vertex.create("Child", ProcessorDescriptor.create(FailingAttemptProcessor.class.getName()), 2);

    OrderedPartitionedKVEdgeConfig edgeConfig = OrderedPartitionedKVEdgeConfig
        .newBuilder(Text.class.getName(), IntWritable.class.getName(),
            HashPartitioner.class.getName())
        .setFromConfiguration(tezConf)
        .build();
    dag.addVertex(vertex1)
        .addVertex(vertex2)
        .addEdge(Edge.create(vertex1, vertex2, edgeConfig.createDefaultEdgeProperty()));

    DAGClient dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  } finally {
    tezClient.stop();
  }
}
 

public static DAG createDAG(String name,
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS, MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);

  // Make each vertex manager fail on appropriate attempt
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      FailOnAttemptVertexManagerPlugin.class.getName())
      .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("1").getBytes()))));
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      FailOnAttemptVertexManagerPlugin.class.getName())
      .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("2").getBytes()))));
  v3.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      FailOnAttemptVertexManagerPlugin.class.getName())
      .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("3").getBytes()))));
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addEdge(Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

protected static void addDAGVerticesAndEdges() {
    l1v1 = Vertex.create("l1v1", TestProcessor.getProcDesc(payload), 1, defaultResource);
    l2v1 = Vertex.create("l2v1", TestProcessor.getProcDesc(payload), 1, defaultResource);
    addVerticesAndEdgeInternal(l1v1, l2v1, DataMovementType.SCATTER_GATHER);
    l1v2 = Vertex.create("l1v2", TestProcessor.getProcDesc(payload), 2, defaultResource);
    l2v2 = Vertex.create("l2v2", TestProcessor.getProcDesc(payload), 3, defaultResource);
    addVerticesAndEdgeInternal(l1v2, l2v2, DataMovementType.SCATTER_GATHER);
    l1v3 = Vertex.create("l1v3", TestProcessor.getProcDesc(payload), 3, defaultResource);
    l2v3 = Vertex.create("l2v3", TestProcessor.getProcDesc(payload), 2, defaultResource);
    addVerticesAndEdgeInternal(l1v3, l2v3, DataMovementType.SCATTER_GATHER);
    l1v4 = Vertex.create("l1v4", TestProcessor.getProcDesc(payload), 2, defaultResource);
    l2v4 = Vertex.create("l2v4", TestProcessor.getProcDesc(payload), 3, defaultResource);
    addVerticesAndEdgeInternal(l1v4, l2v4, DataMovementType.BROADCAST);
}
 

DAG createDAG(DataMovementType dmType) {
  DAG dag = DAG.create("test-" + dagCount++);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
  Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 5);
  Edge eAB = Edge.create(vA, vB, 
  EdgeProperty.create(dmType, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL, OutputDescriptor.create("O.class"),
      InputDescriptor.create("I.class")));
  
  dag.addVertex(vA).addVertex(vB).addEdge(eAB);
  return dag;
}
 

private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName,
                                           Vertex.VertexExecutionContext lhsContext) throws
    TezException,
    InterruptedException, IOException {
  LOG.info("Running JoinValidate with componentName reportNonFatalException");
  JoinValidateConfigured joinValidate =
      new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext,
          EXECUTION_CONTEXT_EXT_SERVICE_PUSH,
          EXECUTION_CONTEXT_EXT_SERVICE_PUSH, componentName);

  DAG dag = joinValidate
      .createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()),
          HASH_JOIN_EXPECTED_RESULT_PATH,
          HASH_JOIN_OUTPUT_PATH, 3);

  DAGClient dagClient = tezClient.submitDAG(dag);

  DAGStatus dagStatus =
      dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
  assertEquals(DAGStatus.State.FAILED, dagStatus.getState());

  boolean foundDiag = false;
  for (String diag : dagStatus.getDiagnostics()) {
    if (diag.contains(ErrorPluginConfiguration.REPORT_NONFATAL_ERROR_MESSAGE) &&
        diag.contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())) {
      foundDiag = true;
      break;
    }
  }
  assertTrue(foundDiag);
}
 

@Override
public DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set<Vertex> vertices,
    @Nullable Set<StatusGetOpts> statusGetOpts) throws IOException, TezException {
  Set<String> vertexNames = new HashSet<String>();
  if (vertices != null) {
    for (Vertex vertex : vertices) {
      vertexNames.add(vertex.getName());
    }
  }
  return _waitForCompletionWithStatusUpdates(vertexNames, statusGetOpts);
}
 

@Test(timeout = 5000)
public void testAMLoggingOptsSimple() throws IOException, YarnException {

  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());

  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  Credentials credentials = new Credentials();
  JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
  TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
  DAG dag = DAG.create("testdag");
  dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
      .setTaskLaunchCmdOpts("initialLaunchOpts"));
  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appSubmissionContext =
      TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
          new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
          null, null);

  List<String> expectedCommands = new LinkedList<String>();
  expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
      ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
      TezConstants.TEZ_CONTAINER_LOGGER_NAME);

  List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
  assertEquals(1, commands.size());
  for (String expectedCmd : expectedCommands) {
    assertTrue(commands.get(0).contains(expectedCmd));
  }

  Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
  String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
  assertNull(logEnv);
}
 

protected static void addDAGVerticesAndEdges() {
    TwoLevelsFailingDAG.addDAGVerticesAndEdges();
    l3v1 = Vertex.create("l3v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l3v1);
    addEdge(l2v1, l3v1, DataMovementType.SCATTER_GATHER);
    addEdge(l2v2, l3v1, DataMovementType.SCATTER_GATHER);
    l3v2 = Vertex.create("l3v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l3v2);
    addEdge(l2v2, l3v2, DataMovementType.BROADCAST);
    addEdge(l2v3, l3v2, DataMovementType.SCATTER_GATHER);
    addEdge(l2v4, l3v2, DataMovementType.SCATTER_GATHER);
}
 

static void setDefaultLaunchCmdOpts(Vertex v, TezConfiguration conf) {
  String vOpts = v.getTaskLaunchCmdOpts();
  String vConfigOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
  if (vConfigOpts != null && vConfigOpts.length() > 0) {
    vOpts += (" " + vConfigOpts);
  }
  
  vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
      TezConfiguration.TEZ_TASK_LOG_LEVEL,
      TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT), vOpts);
  v.setTaskLaunchCmdOpts(vOpts);
}
 

@Test (timeout = 5000)
public void testLocalResourceSetup() throws Exception {
  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  
  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(false);
  
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("testLocalResourceSetup").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  DAGClient dagClient = tezClient.submitDAG(dag);
  mockLauncher.waitTillContainersLaunched();
  ContainerData cData = mockLauncher.getContainers().values().iterator().next();
  ContainerLaunchContext launchContext = cData.launchContext;
  Map<String, LocalResource> taskLR = launchContext.getLocalResources();
  // verify tasks are launched with both DAG and task resources.
  Assert.assertTrue(taskLR.containsKey(lrName1));
  Assert.assertTrue(taskLR.containsKey(lrName2));
  
  mockLauncher.startScheduling(true);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 

@Test (timeout = 10000)
public void testMultipleSubmissions() throws Exception {
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  DAGClient dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
  
  // submit the same DAG again to verify it can be done.
  tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 

private DAG createSimpleDAG(String dagName, String processorName) {
  DAG dag = DAG.create(dagName).addVertex(Vertex.create("Sleep", ProcessorDescriptor.create(
      processorName).setUserPayload(
      new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1));
  return dag;

}
 

private DAG createDAG(TezConfiguration tezConf) throws IOException {
  InputDescriptor inputDescriptor = InputDescriptor.create(FakeInput.class.getName());
  InputInitializerDescriptor inputInitializerDescriptor =
    InputInitializerDescriptor.create(FakeInputInitializer.class.getName());
  DataSourceDescriptor dataSourceDescriptor =
    DataSourceDescriptor.create(inputDescriptor, inputInitializerDescriptor, null);

  Vertex v1 = Vertex.create(VERTEX1, ProcessorDescriptor.create(TokenProcessor.class.getName()));
  v1.addDataSource(INPUT, dataSourceDescriptor);
  Vertex v2 = Vertex.create(VERTEX2, ProcessorDescriptor.create(TokenProcessor.class.getName()));
  v2.addDataSource(INPUT, dataSourceDescriptor);

  OutputDescriptor outputDescriptor = OutputDescriptor.create(FakeOutput.class.getName());
  OutputCommitterDescriptor outputCommitterDescriptor =
    OutputCommitterDescriptor.create(FakeOutputCommitter.class.getName());
  DataSinkDescriptor dataSinkDescriptor =
    DataSinkDescriptor.create(outputDescriptor, outputCommitterDescriptor, null);

  CartesianProductConfig cartesianProductConfig =
    new CartesianProductConfig(Arrays.asList(sourceVertices));
  UserPayload userPayload = cartesianProductConfig.toUserPayload(tezConf);

  Vertex v3 = Vertex.create(VERTEX3, ProcessorDescriptor.create(JoinProcessor.class.getName()));
  v3.addDataSink(OUTPUT, dataSinkDescriptor);
  v3.setVertexManagerPlugin(
    VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName())
                                 .setUserPayload(userPayload));

  EdgeManagerPluginDescriptor edgeManagerDescriptor =
    EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName());
  edgeManagerDescriptor.setUserPayload(userPayload);
  UnorderedPartitionedKVEdgeConfig edgeConf =
    UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(),
      RoundRobinPartitioner.class.getName()).build();
  EdgeProperty edgeProperty = edgeConf.createDefaultCustomEdgeProperty(edgeManagerDescriptor);

  return DAG.create("CrossProduct").addVertex(v1).addVertex(v2).addVertex(v3)
    .addEdge(Edge.create(v1, v3, edgeProperty)).addEdge(Edge.create(v2, v3, edgeProperty));
}