Java Code Examples for org.apache.tez.dag.api.TezConfiguration#set()

private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}
 

/**
 * 
 * @throws Exception
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnored() throws Exception {
  URL[] cp = TezClassLoader.getInstance().getURLs();
  StringBuffer buffer = new StringBuffer();
  for (URL url : cp) {
    buffer.append(url.toExternalForm());
    buffer.append(",");
  }
  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, buffer.toString());
  conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  Assert.assertFalse(TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap));
  assertTrue(localizedMap.isEmpty());
}
 

/**
 * 
 * @throws Exception
 */
@Test (timeout=20000)
public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnoredSetToFalse() throws Exception {
  URL[] cp = TezClassLoader.getInstance().getURLs();
  StringBuffer buffer = new StringBuffer();
  for (URL url : cp) {
    buffer.append(url.toExternalForm());
    buffer.append(",");
  }
  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, buffer.toString());
  conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false);
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  Assert.assertFalse(TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap));
  assertFalse(localizedMap.isEmpty());
}
 

@Test(timeout = 120000)
public void testMultipleMRRSleepJobViaSession() throws IOException,
InterruptedException, TezException, ClassNotFoundException, YarnException {
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
      .valueOf(new Random().nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  TezConfiguration tezConf = new TezConfiguration(
      mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());

  TezClient tezSession = TezClient.create("testsession", tezConf, true);
  tezSession.start();
  Assert.assertEquals(TezAppMasterStatus.INITIALIZING,
      tezSession.getAppMasterStatus());

  State finalState = testMRRSleepJobDagSubmitCore(true, false, false,
      tezSession, false, null, null);
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
  Assert.assertEquals(TezAppMasterStatus.READY,
      tezSession.getAppMasterStatus());
  finalState = testMRRSleepJobDagSubmitCore(true, false, false,
      tezSession, false, null, null);
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
  Assert.assertEquals(TezAppMasterStatus.READY,
      tezSession.getAppMasterStatus());

  stopAndVerifyYarnApp(tezSession);
}
 

@Test(timeout=1000)
public void testSetApplicationTags() {
  TezConfiguration conf = new TezConfiguration(false);
  conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar");
  AMConfiguration amconfig = new AMConfiguration(conf, null, null);
  ApplicationSubmissionContext appContext = Records
      .newRecord(ApplicationSubmissionContext.class);
  Collection<String> tagsFromConf =
      amconfig.getTezConfiguration().getTrimmedStringCollection(
      TezConfiguration.TEZ_APPLICATION_TAGS);
  appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
  assertTrue(appContext.getApplicationTags().contains("foo"));
  assertTrue(appContext.getApplicationTags().contains("bar"));
}
 

private TezConfiguration createConf() {
  TezConfiguration conf = new TezConfiguration();
  conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
  if (useDfs) {
    conf.set("fs.defaultFS", remoteFs.getUri().toString());
  } else {
    conf.set("fs.defaultFS", "file:///");
  }
  conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
  return conf;
}
 

@BeforeClass
public static void setup() throws Exception {
  LOG.info("Starting mini clusters");
  FileSystem remoteFs = null;
  try {
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
        .format(true).racks(null).build();
    remoteFs = dfsCluster.getFileSystem();
  } catch (IOException io) {
    throw new RuntimeException("problem starting mini dfs cluster", io);
  }
  if (miniTezCluster == null) {
    miniTezCluster = new MiniTezCluster(TestFaultTolerance.class.getName(),
        4, 1, 1);
    Configuration miniTezconf = new Configuration(conf);
    miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
    miniTezCluster.init(miniTezconf);
    miniTezCluster.start();
    
    Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
    
    TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());
    tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);

    tezSession = new TezClient("TestFaultTolerance", tezConf, true);
    tezSession.start();
  }
}
 

private TezConfiguration createSessionConfig(Path remoteStagingDir) {
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 10);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  return tezConf;
}
 

/**
 * use mini cluster to verify data do not push to ats when the daglogging flag
 * in dagsubmittedevent is set off
 * @throws Exception
 */
@Test (timeout=50000)
public void testDagLoggingDisabled() throws Exception {
  ATSHistoryLoggingService historyLoggingService;
  historyLoggingService =
      ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
  AppContext appContext = mock(AppContext.class);
  when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
  historyLoggingService.setAppContext(appContext);
  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  String viewAcls = "nobody nobody_group";
  tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      ATSHistoryLoggingService.class.getName());
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
      .nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
  historyLoggingService.init(tezConf);
  historyLoggingService.start();
  ApplicationId appId = ApplicationId.newInstance(100l, 1);
  TezDAGID tezDAGID = TezDAGID.getInstance(
                      appId, 100);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
  DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
  DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID,
        1, dagPlan, appAttemptId, null,
        "usr", tezConf, null, null);
  submittedEvent.setHistoryLoggingEnabled(false);
  DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
  historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
  Thread.sleep(1000l);
  String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDagID();
  Client client = new Client();
  WebResource resource = client.resource(url);

  ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(404, response.getStatus());
}
 

public static void setupTezCluster() throws Exception {
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 3 * 1000);
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 3 * 1000);
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 2);

  //Enable per edge counters
  conf.setBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, true);
  conf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService
      .class.getName());

  conf.set(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR, SIMPLE_HISTORY_DIR);

  miniTezCluster =
      new MiniTezClusterWithTimeline(TEZ_BASE_DIR, 1, 1, 1, true);

  miniTezCluster.init(conf);
  miniTezCluster.start();

  createSampleFile(inputLoc);

  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
      miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS));
  tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      ATSHistoryLoggingService.class.getName());
  yarnTimelineAddress = miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS);

}
 

@Test(timeout = 5000)
public void testAMLoggingOptsSimple() throws IOException, YarnException {

  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());

  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  Credentials credentials = new Credentials();
  JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
  TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
  DAG dag = DAG.create("testdag");
  dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
      .setTaskLaunchCmdOpts("initialLaunchOpts"));
  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appSubmissionContext =
      TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
          new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
          null, null);

  List<String> expectedCommands = new LinkedList<String>();
  expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
      ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
      TezConstants.TEZ_CONTAINER_LOGGER_NAME);

  List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
  assertEquals(1, commands.size());
  for (String expectedCmd : expectedCommands) {
    assertTrue(commands.get(0).contains(expectedCmd));
  }

  Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
  String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
  assertNull(logEnv);
}
 

/**
 * 
 */
@Test(expected=FileNotFoundException.class, timeout=5000)
public void validateSetTezJarLocalResourcesDefinedNonExistingDirectory() throws Exception {

  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, "file:///foo");
  Credentials credentials = new Credentials();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials);
}
 

private void startNonSessionClient() throws Exception {
  LOG.info("Starting Client");
  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
  tezClient = TezClient.create("TestExceptionPropagation", tezConf);
  tezClient.start();
}
 

@BeforeClass
public static void setup() throws Exception {
  LOG.info("Starting mini clusters");
  FileSystem remoteFs = null;
  try {
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
        .format(true).racks(null).build();
    remoteFs = dfsCluster.getFileSystem();
  } catch (IOException io) {
    throw new RuntimeException("problem starting mini dfs cluster", io);
  }
  if (miniTezCluster == null) {
    miniTezCluster = new MiniTezCluster(TestFaultTolerance.class.getName(),
        3, 1, 1);
    Configuration miniTezconf = new Configuration(conf);
    miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
    miniTezCluster.init(miniTezconf);
    miniTezCluster.start();
    
    Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
        .valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
    
    tezConf = new TezConfiguration(miniTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());
    tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    tezConf.setDouble(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES_FRACTION, 0.4);
    tezConf.setInt(TezConfiguration.TEZ_AM_MAX_ALLOWED_TIME_FOR_TASK_READ_ERROR_SEC, 3);
    tezConf.setInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS, 100);
    tezConf.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 500);

    tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
    tezSession.start();
  }
}
 

private void testOrderedWordCountMultipleRoundRecoverying(
        RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition shutdownCondition,
        boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception {

  for (int i=0; i<shutdownCondition.size(); i++) {
    SimpleShutdownCondition condition = shutdownCondition.getSimpleShutdownCondition(i);
    LOG.info("ShutdownCondition:" + condition.getEventType()
            + ", event=" + condition.getEvent());
  }

  String inputDirStr = "/tmp/owc-input/";
  Path inputDir = new Path(inputDirStr);
  Path stagingDirPath = new Path("/tmp/owc-staging-dir");
  remoteFs.mkdirs(inputDir);
  remoteFs.mkdirs(stagingDirPath);
  TestTezJobs.generateOrderedWordCountInput(inputDir, remoteFs);

  String outputDirStr = "/tmp/owc-output/";
  Path outputDir = new Path(outputDirStr);

  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.set(TezConfiguration.TEZ_AM_RECOVERY_SERVICE_CLASS,
          RecoveryServiceWithEventHandlingHook.class.getName());
  tezConf.set(
          RecoveryServiceWithEventHandlingHook.AM_RECOVERY_SERVICE_HOOK_CLASS,
          RecoveryServiceWithEventHandlingHook.MultipleRoundRecoveryEventHook.class.getName());
  tezConf.set(RecoveryServiceWithEventHandlingHook.MultipleRoundRecoveryEventHook.MULTIPLE_ROUND_SHUTDOWN_CONDITION,
          shutdownCondition.serialize());
  tezConf.setBoolean(
          ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
          enableAutoParallelism);
  tezConf.setBoolean(
          RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, false);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  tezConf.setBoolean(
          TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, false);
  OrderedWordCount job = new OrderedWordCount();
  if (generateSplitInClient) {
    Assert
            .assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{
                    "-generateSplitInClient", inputDirStr, outputDirStr, "5"}, null) == 0);
  } else {
    Assert
            .assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{
                    inputDirStr, outputDirStr, "5"}, null) == 0);
  }
  TestTezJobs.verifyOutput(outputDir, remoteFs);
}
 

@Test(timeout = 60000)
public void testHistoryLogging() throws IOException,
    InterruptedException, TezException, ClassNotFoundException, YarnException {
  SleepProcessorConfig spConf = new SleepProcessorConfig(1);

  DAG dag = DAG.create("TezSleepProcessorHistoryLogging");
  Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
          SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 2,
      Resource.newInstance(1024, 1));
  dag.addVertex(vertex);

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
      .nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

  FileSystem localFs = FileSystem.getLocal(tezConf);
  Path historyLogDir = new Path(TEST_ROOT_DIR, "testHistoryLogging");
  localFs.mkdirs(historyLogDir);

  tezConf.set(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR,
      localFs.makeQualified(historyLogDir).toString());

  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
  TezClient tezSession = TezClient.create("TezSleepProcessorHistoryLogging", tezConf);
  tezSession.start();

  DAGClient dagClient = tezSession.submitDAG(dag);

  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
        + dagStatus.getState());
    Thread.sleep(500l);
    dagStatus = dagClient.getDAGStatus(null);
  }
  assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());

  FileStatus historyLogFileStatus = null;
  for (FileStatus fileStatus : localFs.listStatus(historyLogDir)) {
    if (fileStatus.isDirectory()) {
      continue;
    }
    Path p = fileStatus.getPath();
    if (p.getName().startsWith(SimpleHistoryLoggingService.LOG_FILE_NAME_PREFIX)) {
      historyLogFileStatus = fileStatus;
      break;
    }
  }
  Assert.assertNotNull(historyLogFileStatus);
  Assert.assertTrue(historyLogFileStatus.getLen() > 0);
  tezSession.stop();
}
 

@Test(timeout = 5000)
public void testTaskCommandOpts() throws TezException {
  TezConfiguration tezConf = new TezConfiguration();
  String taskCommandOpts = "-Xmx 200m -Dtest.property";
  tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts);
  String expected = null;

  // Test1: Rely on defaults for cluster default opts
  String taskOptsConstructed = TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf);
  expected =
      TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + taskCommandOpts;
  assertTrue(
      "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed +
          "]", taskOptsConstructed.startsWith(expected));

  // Test2: Setup cluster-default command opts explicitly
  String taskClusterDefaultCommandOpts =
      "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc ";
  tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
      taskClusterDefaultCommandOpts);
  taskOptsConstructed =
      TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf);
  expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts;
  assertTrue(
      "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed +
          "]", taskOptsConstructed.startsWith(expected));

  // Test3: Don't setup Xmx explicitly
  taskCommandOpts = "-Dtest.property";
  tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts);
  taskOptsConstructed =
      TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf);
  expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts;
  assertTrue(
      "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed +
          "]", taskOptsConstructed.startsWith(expected));

  // Test4: Pass in a dag-configured value.
  String programmaticTaskOpts = "-Dset.programatically=true -Djava.net.preferIPv4Stack=false";
  taskOptsConstructed =
      TezClientUtils.addDefaultsToTaskLaunchCmdOpts(programmaticTaskOpts, tezConf);
  // Container logging is always added at the end, if it's required.
  expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts + " " + programmaticTaskOpts;
  assertTrue(
      "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed +
          "]", taskOptsConstructed.startsWith(expected));
}
 

private void _testTezClientSessionLargeDAGPlan(int maxIPCMsgSize, int payloadSize, int amResourceSize,
                                             boolean shouldSerialize) throws Exception {
  TezConfiguration conf = new TezConfiguration();
  conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);
  conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, "target/"+this.getClass().getName());
  TezClientForTest client = configureAndCreateTezClient(null, true, conf);

  Map<String, LocalResource> localResourceMap = new HashMap<>();
  byte[] bytes = new byte[amResourceSize];
  Arrays.fill(bytes, (byte)1);
  String lrName = new String(bytes);
  localResourceMap.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("P");
  processorDescriptor.setUserPayload(UserPayload.create(ByteBuffer.allocate(payloadSize)));
  Vertex vertex = Vertex.create("Vertex", processorDescriptor, 1, Resource.newInstance(1, 1));
  DAG dag = DAG.create("DAG").addVertex(vertex);

  client.start();
  client.addAppMasterLocalFiles(localResourceMap);
  client.submitDAG(dag);
  client.stop();

  ArgumentCaptor<SubmitDAGRequestProto> captor = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
  verify(client.sessionAmProxy).submitDAG((RpcController)any(), captor.capture());
  SubmitDAGRequestProto request = captor.getValue();

  if (shouldSerialize) {
    /* we need manually delete the serialized dagplan since staging path here won't be destroyed */
    Path dagPlanPath = new Path(request.getSerializedRequestPath());
    FileSystem fs = FileSystem.getLocal(conf);
    fs.deleteOnExit(dagPlanPath);
    fs.delete(dagPlanPath, false);

    assertTrue(request.hasSerializedRequestPath());
    assertFalse(request.hasDAGPlan());
    assertFalse(request.hasAdditionalAmResources());
  } else {
    assertFalse(request.hasSerializedRequestPath());
    assertTrue(request.hasDAGPlan());
    assertTrue(request.hasAdditionalAmResources());
  }
}
 

/**
 *
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesMixTarballAndJar() throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  StringBuilder tezLibUris = new StringBuilder();

  // Create 2 jars and 1 archive
  Path topDir = new Path(TEST_ROOT_DIR, "validatetarballandjar");
  if (localFs.exists(topDir)) {
    localFs.delete(topDir, true);
  }
  localFs.mkdirs(topDir);

  Path tarFile1 = new Path(topDir, "f1.tar.gz");
  Path jarFile2 = new Path(topDir, "f2.jar");
  Path jarFile3 = new Path(topDir, "f3.jar");

  Assert.assertTrue(localFs.createNewFile(tarFile1));
  Assert.assertTrue(localFs.createNewFile(jarFile2));
  Assert.assertTrue(localFs.createNewFile(jarFile3));

  tezLibUris.append(localFs.makeQualified(topDir).toString()).append(",");
  tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(",");

  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString());
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
  Set<String> resourceNames = localizedMap.keySet();
  Assert.assertEquals(4, resourceNames.size());
  Assert.assertTrue(resourceNames.contains("tar1"));
  Assert.assertTrue(resourceNames.contains("f1.tar.gz"));
  Assert.assertTrue(resourceNames.contains("f2.jar"));
  Assert.assertTrue(resourceNames.contains("f3.jar"));

  Assert.assertTrue(localFs.delete(tarFile1, true));
  Assert.assertTrue(localFs.delete(jarFile2, true));
  Assert.assertTrue(localFs.delete(jarFile3, true));
  Assert.assertTrue(localFs.delete(topDir, true));
}
 

public static TezConfiguration getDAGAMConfFromMRConf(
        Configuration tezConf) {

    // Set Tez parameters based on MR parameters.
    TezConfiguration dagAMConf = new TezConfiguration(tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
            .getMRToDAGParamMap();

    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
        if (dagAMConf.get(entry.getKey()) != null) {
            dagAMConf.set(entry.getValue(), dagAMConf.get(entry.getKey()));
            dagAMConf.unset(entry.getKey());
            if (LOG.isDebugEnabled()) {
                LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
                        + " to Tez key: " + entry.getValue()
                        + " with value " + dagAMConf.get(entry.getValue()));
            }
        }
    }

    String env = tezConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
    if (tezConf.get(MRJobConfig.MR_AM_ENV) != null) {
        env = (env == null) ? tezConf.get(MRJobConfig.MR_AM_ENV)
                            : env + "," + tezConf.get(MRJobConfig.MR_AM_ENV);
    }

    if (env != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
    }

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
            org.apache.tez.mapreduce.hadoop.MRHelpers
                    .getJavaOptsForMRAM(tezConf));

    String queueName = tezConf.get(JobContext.QUEUE_NAME,
            YarnConfiguration.DEFAULT_QUEUE_NAME);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);

    int amMemMB = tezConf.getInt(MRJobConfig.MR_AM_VMEM_MB,
            MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
    int amCores = tezConf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
            MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, ""
            + amMemMB);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, ""
            + amCores);

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""
            + dagAMConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
                    MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

    if (tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY) != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_CREDENTIALS_PATH,
                tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY));
    }

    //TODO: Strip out all MR settings

    return dagAMConf;
}