org.apache.tez.client.TezClient Java Examples

@Override
public JobStatus getJobStatus(JobID jobID) throws IOException,
    InterruptedException {
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  String jobFile = MRApps.getJobFile(conf, user, jobID);
  DAGStatus dagStatus;
  try {
    if(dagClient == null) {
      dagClient = TezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId(), tezConf);
    }
    dagStatus = dagClient.getDAGStatus(null);
    return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
  } catch (TezException e) {
    throw new IOException(e);
  }
}
 

void runDAGAndVerify(DAG dag, DAGStatus.State finalState,
                     TezClient session) throws Exception {
  session.waitTillReady();
  DAGClient dagClient = session.submitDAG(dag);
  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for dag to complete. Sleeping for 500ms."
        + " DAG name: " + dag.getName()
        + " DAG appId: " + dagClient.getApplicationId()
        + " Current state: " + dagStatus.getState());
    Thread.sleep(100);
    dagStatus = dagClient.getDAGStatus(null);
  }

  Assert.assertEquals(finalState, dagStatus.getState());
}
 

@Test(timeout=120000)
public void testSessionDisableMultiAttempts() throws Exception {
  tezSession.stop();
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
  TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  TezClient session = new TezClient("TestDAGRecovery2SingleAttemptOnly", tezConf);
  session.start();

  // DAG should fail as it never completes on the first attempt
  DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
  runDAGAndVerify(dag, State.FAILED, session);
  session.stop();
}
 

@Test(timeout=120000)
public void testSessionDisableMultiAttempts() throws Exception {
  tezSession.stop();
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
  TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  TezClient session = TezClient.create("TestDAGRecovery2SingleAttemptOnly", tezConf);
  session.start();

  // DAG should fail as it never completes on the first attempt
  DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
  runDAGAndVerify(dag, State.FAILED, session);
  session.stop();
}
 

/**
 * Utility method to use the example from within code or a test.
 *
 * @param conf      the tez configuration instance which will be used to crate the DAG and
 *                  possible the Tez Client.
 * @param args      arguments to the example
 * @param tezClient an existing running {@link org.apache.tez.client.TezClient} instance if one
 *                  exists. If no TezClient is specified (null), one will be created based on the
 *                  provided configuration. If TezClient is specified, local mode option can not been
 *                  specified in arguments, it takes no effect.
 * @return Zero indicates success, non-zero indicates failure
 * @throws Exception 
 */
public int run(TezConfiguration conf, String[] args, @Nullable TezClient tezClient) throws
    Exception {
  setConf(conf);
  hadoopShim = new HadoopShimsLoader(conf).getHadoopShim();
  GenericOptionsParser optionParser = new GenericOptionsParser(conf, getExtraOptions(), args);
  if (optionParser.getCommandLine().hasOption(LOCAL_MODE)) {
    isLocalMode = true;
    if (tezClient != null) {
      throw new RuntimeException("can't specify local mode when TezClient is created, it takes no effect");
    }
  }
  if (optionParser.getCommandLine().hasOption(DISABLE_SPLIT_GROUPING)) {
    disableSplitGrouping = true;
  }
  if (optionParser.getCommandLine().hasOption(COUNTER_LOG)) {
    isCountersLog = true;
  }
  if (optionParser.getCommandLine().hasOption(GENERATE_SPLIT_IN_CLIENT)) {
    generateSplitInClient = true;
  }
  String[] otherArgs = optionParser.getRemainingArgs();
  return _execute(otherArgs, conf, tezClient);
}
 

@Test(timeout = 20000)
public void testNoSysExitOnFailinglDAG() throws TezException, InterruptedException,
    IOException {
  TezConfiguration tezConf1 = createConf();
  // Run in non-session mode so that the AM terminates
  TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
  tezClient1.start();

  DAG dag1 = createSimpleDAG("dag1", FailingProcessor.class.getName());

  DAGClient dagClient1 = tezClient1.submitDAG(dag1);
  dagClient1.waitForCompletion();
  assertEquals(DAGStatus.State.FAILED, dagClient1.getDAGStatus(null).getState());

  // Sleep for more time than is required for the DAG to complete.
  Thread.sleep((long) (TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT * 1.5));

  dagClient1.close();
  tezClient1.stop();
}
 

@Test(timeout = 20000)
public void testNoSysExitOnSuccessfulDAG() throws TezException, InterruptedException,
    IOException {
  TezConfiguration tezConf1 = createConf();
  // Run in non-session mode so that the AM terminates
  TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
  tezClient1.start();

  DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());

  DAGClient dagClient1 = tezClient1.submitDAG(dag1);
  dagClient1.waitForCompletion();
  assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());

  // Sleep for more time than is required for the DAG to complete.
  Thread.sleep((long) (TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT * 1.5));

  dagClient1.close();
  tezClient1.stop();
}
 

@Test(timeout = 20000)
public void testFatalErrorReported() throws IOException, TezException, InterruptedException {

  TezClient tezClient = getTezClient("testFatalErrorReported");
  DAGClient dagClient = null;

  try {
    FailingProcessor.configureForFatalFail();
    DAG dag = DAG.create("testFatalErrorReportedDag").addVertex(
        Vertex
            .create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));

    dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
    assertEquals(1, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount());
  } finally {
    if (dagClient != null) {
      dagClient.close();
    }
    tezClient.stop();
  }
}
 

TezClient getTezClient(boolean withTimeline) throws Exception {
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  if (withTimeline) {
    tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, withTimeline);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
        ATSHistoryLoggingService.class.getName());
  } else {
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
        SimpleHistoryLoggingService.class.getName());
  }
  tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);

  TezClient tezClient = TezClient.create("WordCount", tezConf, false);
  tezClient.start();
  tezClient.waitTillReady();
  return tezClient;
}
 

private static SessionInfo createSession(Configuration conf,
        Map<String, LocalResource> requestedAMResources, Credentials creds,
        TezJobConfig tezJobConf) throws TezException, IOException,
        InterruptedException {
    TezConfiguration amConf = MRToTezHelper.getDAGAMConfFromMRConf(conf);
    TezScriptState ss = TezScriptState.get();
    ss.addDAGSettingsToConf(amConf);
    adjustAMConfig(amConf, tezJobConf);
    String jobName = conf.get(PigContext.JOB_NAME, "pig");
    TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds);
    tezClient.start();
    TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus();
    if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) {
        throw new RuntimeException("TezSession has already shutdown");
    }
    tezClient.waitTillReady();
    return new SessionInfo(tezClient, requestedAMResources);
}
 

@Before
public void setup()  throws Exception {
  LOG.info("Starting session");
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, "false");
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000);

  tezSession = TezClient.create("TestDAGRecovery", tezConf);
  tezSession.start();
}
 

private void startSessionClient() throws Exception {
  LOG.info("Starting session");
  tezConf = new TezConfiguration();
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setBoolean(TezConfiguration.TEZ_AM_ONE_TO_ONE_ROUTING_USE_ON_DEMAND_ROUTING, true);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  // for local mode
  tezConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
  tezConf.set("fs.defaultFS", "file:///");
  tezConf.setBoolean(
      TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);

  tezSession = TezClient.create("TestExceptionPropagation", tezConf);
  tezSession.start();
}
 

@Test (timeout = 150000)
public void testNonFatalErrors() throws IOException, TezException, InterruptedException {
  String methodName = "testNonFatalErrors";
  TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
  TezClient tezClient = TezClient
      .newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session",
          tezClientConf)
      .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
  try {
    tezClient.start();
    LOG.info("TezSessionStarted for " + methodName);
    tezClient.waitTillReady();
    LOG.info("TezSession ready for submission for " + methodName);


    runAndVerifyForNonFatalErrors(tezClient, SUFFIX_LAUNCHER, EXECUTION_CONTEXT_LAUNCHER_REPORT_NON_FATAL);
    runAndVerifyForNonFatalErrors(tezClient, SUFFIX_TASKCOMM, EXECUTION_CONTEXT_TASKCOMM_REPORT_NON_FATAL);
    runAndVerifyForNonFatalErrors(tezClient, SUFFIX_SCHEDULER, EXECUTION_CONTEXT_SCHEDULER_REPORT_NON_FATAL);

  } finally {
    tezClient.stop();
  }
}
 

@Test(timeout = 60000)
public void testMultipleCommits_OnVertexSuccess() throws Exception {
  Path stagingDirPath = new Path("/tmp/commit-staging-dir");
  Random rand = new Random();
  String v1OutputPathPrefix = "/tmp/commit-output-v1";
  int v1OutputNum = rand.nextInt(10) + 1;
  String v2OutputPathPrefix = "/tmp/commit-output-v2";
  int v2OutputNum = rand.nextInt(10) + 1;
  String uv12OutputPathPrefix = "/tmp/commit-output-uv12";
  int uv12OutputNum = rand.nextInt(10) + 1;
  String v3OutputPathPrefix = "/tmp/commit-output-v3";
  int v3OutputNum = rand.nextInt(10) + 1;
  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  TezClient tezSession = null;

  try {
    MultipleCommitsExample job = new MultipleCommitsExample();
    Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf,
        new String[]{ v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "",
        uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + "",
        MultipleCommitsExample.CommitOnVertexSuccessOption}, null)==0);
    verifyCommits(v1OutputPathPrefix, v1OutputNum);
    verifyCommits(v2OutputPathPrefix, v2OutputNum);
    verifyCommits(uv12OutputPathPrefix, uv12OutputNum);
    verifyCommits(v3OutputPathPrefix, v3OutputNum);
  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (tezSession != null) {
      tezSession.stop();
    }
  }
}
 

@Test(timeout = 30000)
public void testMultipleClientsWithSession() throws TezException, InterruptedException,
    IOException {
  TezConfiguration tezConf1 = createConf();
  TezClient tezClient1 = TezClient.create("commonName", tezConf1, true);
  tezClient1.start();

  DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());

  DAGClient dagClient1 = tezClient1.submitDAG(dag1);
  dagClient1.waitForCompletion();
  assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());

  dagClient1.close();
  tezClient1.stop();

  TezConfiguration tezConf2 = createConf();
  DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
  TezClient tezClient2 = TezClient.create("commonName", tezConf2, true);
  tezClient2.start();
  DAGClient dagClient2 = tezClient2.submitDAG(dag2);
  dagClient2.waitForCompletion();
  assertEquals(DAGStatus.State.SUCCEEDED, dagClient2.getDAGStatus(null).getState());
  assertFalse(dagClient1.getExecutionContext().equals(dagClient2.getExecutionContext()));
  dagClient2.close();
  tezClient2.stop();
}
 

private void startNonSessionClient() throws Exception {
  LOG.info("Starting Client");
  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
  tezClient = TezClient.create("TestExceptionPropagation", tezConf);
  tezClient.start();
}
 

@Test(timeout = 60000)
public void testOrderedWordCount() throws Exception {
  String inputDirStr = "/tmp/owc-input/";
  Path inputDir = new Path(inputDirStr);
  Path stagingDirPath = new Path("/tmp/owc-staging-dir");
  remoteFs.mkdirs(inputDir);
  remoteFs.mkdirs(stagingDirPath);
  generateOrderedWordCountInput(inputDir, remoteFs);

  String outputDirStr = "/tmp/owc-output/";
  Path outputDir = new Path(outputDirStr);

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  TezClient tezSession = null;

  try {

    OrderedWordCount job = new OrderedWordCount();
    Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter", inputDirStr, outputDirStr, "2"}, null)==0);
    verifyOutput(outputDir, remoteFs);

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (tezSession != null) {
      tezSession.stop();
    }
  }

}
 

@Test(timeout = 60000)
public void testOrderedWordCountDisableSplitGrouping() throws Exception {
  String inputDirStr = TEST_ROOT_DIR + "/tmp/owc-input/";
  Path inputDir = new Path(inputDirStr);
  Path stagingDirPath = new Path(TEST_ROOT_DIR + "/tmp/owc-staging-dir");
  localFs.mkdirs(inputDir);
  localFs.mkdirs(stagingDirPath);
  generateOrderedWordCountInput(inputDir, localFs);

  String outputDirStr = TEST_ROOT_DIR + "/tmp/owc-output/";
  localFs.delete(new Path(outputDirStr), true);
  Path outputDir = new Path(outputDirStr);

  TezConfiguration tezConf = new TezConfiguration(conf);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  TezClient tezSession = null;

  try {

    OrderedWordCount job = new OrderedWordCount();
    Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter", "-local", "-disableSplitGrouping",
        inputDirStr, outputDirStr, "2"}, null)==0);
    verifyOutput(outputDir, localFs);

  } finally {
    localFs.delete(stagingDirPath, true);
    if (tezSession != null) {
      tezSession.stop();
    }
  }

}
 

@Test(timeout = 60000)
public void testInputInitializerEvents() throws TezException, InterruptedException, IOException {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  TezClient tezClient = TezClient.create("TestInputInitializerEvents", tezConf);
  tezClient.start();

  try {
    DAG dag = DAG.create("TestInputInitializerEvents");
    Vertex vertex1 = Vertex.create(VERTEX_WITH_INITIALIZER_NAME, ProcessorDescriptor.create(
        SleepProcessor.class.getName())
        .setUserPayload(new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1)
        .addDataSource(INPUT1_NAME,
            DataSourceDescriptor
                .create(InputDescriptor.create(MultiAttemptDAG.NoOpInput.class.getName()),
                    InputInitializerDescriptor.create(InputInitializerForTest.class.getName()),
                    null));
    Vertex vertex2 = Vertex.create(EVENT_GENERATING_VERTEX_NAME,
        ProcessorDescriptor.create(InputInitializerEventGeneratingProcessor.class.getName()), 5);

    dag.addVertex(vertex1).addVertex(vertex2);

    DAGClient dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  } finally {
    tezClient.stop();
  }
}
 

@Test(timeout = 60000)
public void testMultipleCommits_OnDAGSuccess() throws Exception {
  Path stagingDirPath = new Path("/tmp/commit-staging-dir");
  Random rand = new Random();
  String v1OutputPathPrefix = "/tmp/commit-output-v1";
  int v1OutputNum = rand.nextInt(10) + 1;
  String v2OutputPathPrefix = "/tmp/commit-output-v2";
  int v2OutputNum = rand.nextInt(10) + 1;
  String uv12OutputPathPrefix = "/tmp/commit-output-uv12";
  int uv12OutputNum = rand.nextInt(10) + 1;
  String v3OutputPathPrefix = "/tmp/commit-output-v3";
  int v3OutputNum = rand.nextInt(10) + 1;
  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  TezClient tezSession = null;

  try {
    MultipleCommitsExample job = new MultipleCommitsExample();
    Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf,
        new String[]{ v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "",
        uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + ""}, null)==0);
    verifyCommits(v1OutputPathPrefix, v1OutputNum);
    verifyCommits(v2OutputPathPrefix, v2OutputNum);
    verifyCommits(uv12OutputPathPrefix, uv12OutputNum);
    verifyCommits(v3OutputPathPrefix, v3OutputNum);
  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (tezSession != null) {
      tezSession.stop();
    }
  }
}
 

public int run(Configuration conf, String[] args, TezClient tezClient) throws Exception {
    setConf(conf);
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    int result = validateArgs(otherArgs);
    if (result != 0) {
        return result;
    }
    return execute(otherArgs, tezClient);
}
 

private TezClient createTezSession() throws IOException, TezException {
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
      .valueOf(new Random().nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

  TezClient tezSession = TezClient.create("testrelocalizationsession", tezConf, true);
  tezSession.start();
  Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus());
  return tezSession;
}
 

@Override
protected final int runJob(String[] args, TezConfiguration tezConf, TezClient tezClient) throws
    TezException, InterruptedException, IOException {
  LOG.info("Running: " +
      this.getClass().getSimpleName());
  String mode = VIA_RPC;
  if (args.length == 4) {
    if (args[3].equals(VIA_RPC) || args[3].equals(VIA_HDFS_DIRECT_READ) ||
        args[3].equals(VIA_HDFS_DIST_CACHE)) {
      mode = args[3];
    } else {
      printUsage();
      return 2;
    }
  }

  int numTasks = Integer.parseInt(args[0]);
  int maxSleepTimeMillis = Integer.parseInt(args[1]);
  int payloadSizeBytes = Integer.parseInt(args[2]);
  LOG.info("Parameters: numTasks=" + numTasks + ", maxSleepTime(ms)=" + maxSleepTimeMillis +
      ", payloadSize(bytes)=" + payloadSizeBytes + ", mode=" + mode);

  DAG dag = createDAG(tezConf, numTasks, maxSleepTimeMillis, payloadSizeBytes, mode);
  try {
    return runDag(dag, false, LOG);
  } finally {
    if (fs != null) {
      if (resourcePath != null) {
        fs.delete(resourcePath, false);
      }
    }
  }
}
 

private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName,
                                           Vertex.VertexExecutionContext lhsContext) throws
    TezException,
    InterruptedException, IOException {
  LOG.info("Running JoinValidate with componentName reportNonFatalException");
  JoinValidateConfigured joinValidate =
      new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext,
          EXECUTION_CONTEXT_EXT_SERVICE_PUSH,
          EXECUTION_CONTEXT_EXT_SERVICE_PUSH, componentName);

  DAG dag = joinValidate
      .createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()),
          HASH_JOIN_EXPECTED_RESULT_PATH,
          HASH_JOIN_OUTPUT_PATH, 3);

  DAGClient dagClient = tezClient.submitDAG(dag);

  DAGStatus dagStatus =
      dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
  assertEquals(DAGStatus.State.FAILED, dagStatus.getState());

  boolean foundDiag = false;
  for (String diag : dagStatus.getDiagnostics()) {
    if (diag.contains(ErrorPluginConfiguration.REPORT_NONFATAL_ERROR_MESSAGE) &&
        diag.contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())) {
      foundDiag = true;
      break;
    }
  }
  assertTrue(foundDiag);
}
 

private TezClient createTezClient(TezConfiguration tezConf) throws IOException, TezException {
  TezClient tezClient = TezClient.create("TezExampleApplication", tezConf);
  if(reconnectAppId != null) {
    ApplicationId appId = TezClient.appIdfromString(reconnectAppId);
    tezClient.getClient(appId);
  } else {
    tezClient.start();
  }
  return tezClient;
}
 

public void setupSharedTezClient(ServicePluginsDescriptor servicePluginsDescriptor) throws
    IOException, TezException, InterruptedException {
  // Create a session to use for all tests.
  TezConfiguration tezClientConf = new TezConfiguration(confForJobs);

  sharedTezClient = TezClient
      .newBuilder(TestExternalTezServices.class.getSimpleName() + "_session", tezClientConf)
      .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();

  sharedTezClient.start();
  LOG.info("Shared TezSession started");
  sharedTezClient.waitTillReady();
  LOG.info("Shared TezSession ready for submission");
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {
  DAG dag = createDAG(tezConf, args[0], args[1],
      args.length == 3 ? Integer.parseInt(args[2]) : 1);
  LOG.info("Running WordCount");
  return runDag(dag, isCountersLog(), LOG);
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  String inputDir1 = args[0];
  String inputDir2 = args[1];
  int numPartitions = Integer.parseInt(args[2]);
  String outputDir = args[3];

  Path inputPath1 = new Path(inputDir1);
  Path inputPath2 = new Path(inputDir2);
  Path outputPath = new Path(outputDir);

  // Verify output path existence
  FileSystem fs = outputPath.getFileSystem(tezConf);
  outputPath = fs.makeQualified(outputPath);
  if (fs.exists(outputPath)) {
    System.err.println("Output directory: " + outputDir + " already exists");
    return 3;
  }
  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }
  DAG dag =
      createDag(tezConf, inputPath1, inputPath2, outputPath, numPartitions);
  LOG.info("Running SortMergeJoinExample");
  return runDag(dag, isCountersLog(), LOG);
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {
  DAG dag = createDAG(tezConf, args[1], args[2],
      args[3], args[4], args[0].equals(PARTITIONED));
  return runDag(dag, isCountersLog(), LOG);
}
 

@Test(timeout = 60000)
public void testVertexFailuresMaxPercent() throws TezException, InterruptedException, IOException {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, "50.0f");
  tezConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
  TezClient tezClient = TezClient.create("TestVertexFailuresMaxPercent", tezConf);
  tezClient.start();

  try {
    DAG dag = DAG.create("TestVertexFailuresMaxPercent");
    Vertex vertex1 = Vertex.create("Parent", ProcessorDescriptor.create(
        FailingAttemptProcessor.class.getName()), 2);
    Vertex vertex2 = Vertex.create("Child", ProcessorDescriptor.create(FailingAttemptProcessor.class.getName()), 2);

    OrderedPartitionedKVEdgeConfig edgeConfig = OrderedPartitionedKVEdgeConfig
        .newBuilder(Text.class.getName(), IntWritable.class.getName(),
            HashPartitioner.class.getName())
        .setFromConfiguration(tezConf)
        .build();
    dag.addVertex(vertex1)
        .addVertex(vertex2)
        .addEdge(Edge.create(vertex1, vertex2, edgeConfig.createDefaultEdgeProperty()));

    DAGClient dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  } finally {
    tezClient.stop();
  }
}