Java Code Examples for org.apache.tez.client.TezClient#create()

private void createTezSessionATS() throws Exception {
  TezConfiguration tezConf = createCommonTezLog();
  tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
      miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS));
  tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      ATSHistoryLoggingService.class.getName());
  
  Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);

  tezSession = TezClient.create("TestAnalyzer", tezConf, true);
  tezSession.start();
}
 

private void createTezSessionSimpleHistory() throws Exception {
  TezConfiguration tezConf = createCommonTezLog();
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      SimpleHistoryLoggingService.class.getName());

  tezConf.set(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR, SIMPLE_HISTORY_DIR);

  Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);

  tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
  tezSession.start();
}
 

@Test(timeout = 20000)
public void testNoSysExitOnSuccessfulDAG() throws TezException, InterruptedException,
    IOException {
  TezConfiguration tezConf1 = createConf();
  // Run in non-session mode so that the AM terminates
  TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
  tezClient1.start();

  DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());

  DAGClient dagClient1 = tezClient1.submitDAG(dag1);
  dagClient1.waitForCompletion();
  assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());

  // Sleep for more time than is required for the DAG to complete.
  Thread.sleep((long) (TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT * 1.5));

  dagClient1.close();
  tezClient1.stop();
}
 

@Test(timeout = 20000)
public void testNoSysExitOnFailinglDAG() throws TezException, InterruptedException,
    IOException {
  TezConfiguration tezConf1 = createConf();
  // Run in non-session mode so that the AM terminates
  TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
  tezClient1.start();

  DAG dag1 = createSimpleDAG("dag1", FailingProcessor.class.getName());

  DAGClient dagClient1 = tezClient1.submitDAG(dag1);
  dagClient1.waitForCompletion();
  assertEquals(DAGStatus.State.FAILED, dagClient1.getDAGStatus(null).getState());

  // Sleep for more time than is required for the DAG to complete.
  Thread.sleep((long) (TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT * 1.5));

  dagClient1.close();
  tezClient1.stop();
}
 

@Before
public void setup()  throws Exception {
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

  TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
  
  tezSession = TezClient.create("TestDAGRecovery2", tezConf);
  tezSession.start();
}
 

private TezClient createTezClient(TezConfiguration tezConf) throws IOException, TezException {
  TezClient tezClient = TezClient.create("TezExampleApplication", tezConf);
  if(reconnectAppId != null) {
    ApplicationId appId = TezClient.appIdfromString(reconnectAppId);
    tezClient.getClient(appId);
  } else {
    tezClient.start();
  }
  return tezClient;
}
 

@Test(timeout = 60000)
public void testSleepJob() throws TezException, IOException, InterruptedException {
  SleepProcessorConfig spConf = new SleepProcessorConfig(1);

  DAG dag = DAG.create("TezSleepProcessor");
  Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
          SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
      Resource.newInstance(1024, 1));
  dag.addVertex(vertex);

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
      .nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

  TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
  tezSession.start();

  DAGClient dagClient = tezSession.submitDAG(dag);

  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
        + dagStatus.getState());
    Thread.sleep(500l);
    dagStatus = dagClient.getDAGStatus(null);
  }
  dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));

  assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
  assertNotNull(dagStatus.getDAGCounters());
  assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName()));
  assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS));
  ExampleDriver.printDAGStatus(dagClient, new String[] { "SleepVertex" }, true, true);
  tezSession.stop();
}
 

@Test(timeout = 60000)
public void testNonDefaultFSStagingDir() throws Exception {
  SleepProcessorConfig spConf = new SleepProcessorConfig(1);

  DAG dag = DAG.create("TezSleepProcessor");
  Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
          SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
      Resource.newInstance(1024, 1));
  dag.addVertex(vertex);

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  Path stagingDir = new Path(TEST_ROOT_DIR, "testNonDefaultFSStagingDir"
      + String.valueOf(random.nextInt(100000)));
  FileSystem localFs = FileSystem.getLocal(tezConf);
  stagingDir = localFs.makeQualified(stagingDir);
  localFs.mkdirs(stagingDir);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());

  TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
  tezSession.start();

  DAGClient dagClient = tezSession.submitDAG(dag);

  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
        + dagStatus.getState());
    Thread.sleep(500l);
    dagStatus = dagClient.getDAGStatus(null);
  }
  dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));

  assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
  assertNotNull(dagStatus.getDAGCounters());
  assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName()));
  assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS));
  ExampleDriver.printDAGStatus(dagClient, new String[] { "SleepVertex" }, true, true);
  tezSession.stop();
}
 

@Test(timeout = 10000)
public void testMultipleClientsWithoutSession() throws TezException, InterruptedException,
    IOException {
  TezConfiguration tezConf1 = createConf();
  TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
  tezClient1.start();

  DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());

  DAGClient dagClient1 = tezClient1.submitDAG(dag1);
  dagClient1.waitForCompletion();
  assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());

  dagClient1.close();
  tezClient1.stop();


  TezConfiguration tezConf2 = createConf();
  DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
  TezClient tezClient2 = TezClient.create("commonName", tezConf2, false);
  tezClient2.start();
  DAGClient dagClient2 = tezClient2.submitDAG(dag2);
  dagClient2.waitForCompletion();
  assertEquals(DAGStatus.State.SUCCEEDED, dagClient2.getDAGStatus(null).getState());
  assertFalse(dagClient1.getExecutionContext().equals(dagClient2.getExecutionContext()));
  dagClient2.close();
  tezClient2.stop();
}
 

private TezClient createTezSession() throws IOException, TezException {
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
      .valueOf(new Random().nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

  TezClient tezSession = TezClient.create("testrelocalizationsession", tezConf, true);
  tezSession.start();
  Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus());
  return tezSession;
}
 

@BeforeClass
public static void setup() throws Exception {
  LOG.info("Starting mini clusters");
  FileSystem remoteFs = null;
  try {
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
        .format(true).racks(null).build();
    remoteFs = dfsCluster.getFileSystem();
  } catch (IOException io) {
    throw new RuntimeException("problem starting mini dfs cluster", io);
  }
  if (miniTezCluster == null) {
    miniTezCluster = new MiniTezCluster(TestFaultTolerance.class.getName(),
        3, 1, 1);
    Configuration miniTezconf = new Configuration(conf);
    miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
    miniTezCluster.init(miniTezconf);
    miniTezCluster.start();
    
    Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
    
    tezConf = new TezConfiguration(miniTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());
    tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    tezConf.setDouble(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES_FRACTION, 0.4);
    tezConf.setInt(TezConfiguration.TEZ_AM_MAX_ALLOWED_TIME_FOR_TASK_READ_ERROR_SEC, 3);
    tezConf.setInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS, 100);
    tezConf.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 500);

    tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
    tezSession.start();
  }
}
 

@Test (timeout=600000)
public void testSessionStopped() throws Exception {
  Configuration testConf = new Configuration(false);

  testConf.setBoolean(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0);

  // verify value at v2 task1
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "1");

  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4);
  DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf);
  tezSession.waitTillReady();

  DAGClient dagClient = tezSession.submitDAG(dag);
  dagClient.waitForCompletion();
  // kill the session now
  tezSession.stop();

  // Check if killing DAG does not throw any exception
  dagClient.tryKillDAG();

  // restart the session for rest of the tests
  tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
  tezSession.start();
}
 

@Before
public void setup() throws Exception {
  LOG.info("Starting session");
  Path remoteStagingDir =
      remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
          .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, MAX_AM_ATTEMPT);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(
      TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, false);
  tezConf.setBoolean(
      RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED,
      true);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000);
  tezSession = TezClient.create("TestDAGRecovery", tezConf);
  tezSession.start();
}
 

@Test (timeout=50000)
public void testDisabledACls() throws Exception {
  TezClient tezSession = null;
  try {
    SleepProcessorConfig spConf = new SleepProcessorConfig(1);

    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(256, 1));
    dag.addVertex(vertex);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
        ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

    tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
    tezSession.start();

    DAGClient dagClient = tezSession.submitDAG(dag);

    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());
      Thread.sleep(500l);
      dagStatus = dagClient.getDAGStatus(null);
    }
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
  } finally {
    if (tezSession != null) {
      tezSession.stop();
    }
  }
}
 

private void startNonSessionClient() throws Exception {
  LOG.info("Starting Client");
  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
  tezClient = TezClient.create("TestExceptionPropagation", tezConf);
  tezClient.start();
}
 

/**
 * test whole {@link HashJoinExample} pipeline as following: <br>
 * {@link JoinDataGen} -> {@link HashJoinExample} -> {@link JoinValidate}
 * @throws Exception
 */
@Test(timeout = 120000)
public void testHashJoinExampleWithDataViaEvent() throws Exception {

  Path testDir = new Path("/tmp/testHashJoinExampleDataViaEvent");
  Path stagingDirPath = new Path("/tmp/tez-staging-dir");
  remoteFs.mkdirs(stagingDirPath);
  remoteFs.mkdirs(testDir);

  Path dataPath1 = new Path(testDir, "inPath1");
  Path dataPath2 = new Path(testDir, "inPath2");
  Path expectedOutputPath = new Path(testDir, "expectedOutputPath");
  Path outPath = new Path(testDir, "outPath");

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());

  //turn on the dataViaEvent
  tezConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED, true);

  TezClient tezSession = null;
  try {
    tezSession = TezClient.create("HashJoinExampleSession", tezConf, true);
    tezSession.start();

    JoinDataGen dataGen = new JoinDataGen();
    String[] dataGenArgs = new String[] {
            "-counter",
            dataPath1.toString(), "1048576", dataPath2.toString(), "8",
            expectedOutputPath.toString(), "2" };
    assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezSession));

    HashJoinExample joinExample = new HashJoinExample();
    String[] args = new String[] {
            dataPath1.toString(), dataPath2.toString(), "1", outPath.toString(),
            "doBroadcast"};

    assertEquals(0, joinExample.run(tezConf, args, tezSession));

    JoinValidate joinValidate = new JoinValidate();
    String[] validateArgs = new String[] {
            "-counter", expectedOutputPath.toString(), outPath.toString(), "3" };
    assertEquals(0, joinValidate.run(tezConf, validateArgs, tezSession));
  } finally {
    if (tezSession != null) {
      tezSession.stop();
    }
  }
}
 

@Test(timeout = 60000)
public void testAMClientHeartbeatTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/timeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testAMClientHeartbeatTimeout", tezConf, true);
    tezClient.start();
    tezClient.cancelAMKeepAlive(true);

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 

private TezClient createTezClient(TezConfiguration tezConf) throws TezException, IOException {
    TezClient tezClient = TezClient.create("TopK datagen", tezConf);
    tezClient.start();
    return tezClient;
}
 

@Test
public void testATSLogLevelNone() throws Exception {
  TezClient tezSession = null;
  ApplicationId applicationId;
  String viewAcls = "nobody nobody_group";
  try {
    SleepProcessorConfig spConf = new SleepProcessorConfig(1);

    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(256, 1));
    dag.addVertex(vertex);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());

    tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES,
        "TEZ_DAG_ID");

    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
        ATSV15HistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

    tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
    tezSession.start();

    applicationId = tezSession.getAppMasterApplicationId();
    dag.setHistoryLogLevel(HistoryLogLevel.NONE);

    DAGClient dagClient = tezSession.submitDAG(dag);

    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());
      Thread.sleep(500l);
      dagStatus = dagClient.getDAGStatus(null);
    }
    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());

    // Verify HDFS data
    int count = verifyATSDataOnHDFS(atsActivePath, applicationId);
    Assert.assertEquals("Count is: " + count, 1, count);
  } finally {
    if (tezSession != null) {
      tezSession.stop();
    }
  }
}
 

/**
 * Test Disable Logging for all dags in a session 
 * due to failure to create domain in session start
 * @throws Exception
 */
@Test (timeout=50000)
public void testDisableSessionLogging() throws Exception {
  TezClient tezSession = null;
  String viewAcls = "nobody nobody_group";
  SleepProcessorConfig spConf = new SleepProcessorConfig(1);

  DAG dag = DAG.create("TezSleepProcessor");
  Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
          SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
          Resource.newInstance(256, 1));
  dag.addVertex(vertex);
  DAGAccessControls accessControls = new DAGAccessControls();
  accessControls.setUsersWithViewACLs(Collections.singleton("nobody2"));
  accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group2"));
  dag.setAccessControls(accessControls);

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      ATSHistoryLoggingService.class.getName());
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
      .nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

  tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
  tezSession.start();

  //////submit first dag //////
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
        + dagStatus.getState());
    Thread.sleep(500l);
    dagStatus = dagClient.getDAGStatus(null);
  }
  assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());

  //////submit second dag//////
  DAG dag2 = DAG.create("TezSleepProcessor2");
  vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
        SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
     Resource.newInstance(256, 1));
  dag2.addVertex(vertex);
  accessControls = new DAGAccessControls();
  accessControls.setUsersWithViewACLs(Collections.singleton("nobody3"));
  accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group3"));
  dag2.setAccessControls(accessControls);
  dagClient = tezSession.submitDAG(dag2);
  dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
              + dagStatus.getState());
    Thread.sleep(500l);
    dagStatus = dagClient.getDAGStatus(null);
  }
  tezSession.stop();
}