Java Code Examples for org.apache.tez.dag.api.TezConfiguration#setBoolean()

private void createTezSessionSimpleHistory() throws Exception {
  TezConfiguration tezConf = createCommonTezLog();
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      SimpleHistoryLoggingService.class.getName());

  tezConf.set(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR, SIMPLE_HISTORY_DIR);

  Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);

  tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
  tezSession.start();
}
 

@Test(timeout=120000)
public void testSessionDisableMultiAttempts() throws Exception {
  tezSession.stop();
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
  TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  TezClient session = new TezClient("TestDAGRecovery2SingleAttemptOnly", tezConf);
  session.start();

  // DAG should fail as it never completes on the first attempt
  DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
  runDAGAndVerify(dag, State.FAILED, session);
  session.stop();
}
 

@Before
public void setup()  throws Exception {
  LOG.info("Starting session");
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);

  tezSession = new TezClient("TestDAGRecovery", tezConf);
  tezSession.start();
}
 

@Before
public void setup()  throws Exception {
  LOG.info("Starting session");
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, "false");
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000);

  tezSession = TezClient.create("TestDAGRecovery", tezConf);
  tezSession.start();
}
 

@Test
public void testRunNewCombiner() throws IOException, InterruptedException {
  TezConfiguration conf = new TezConfiguration();
  setKeyAndValueClassTypes(conf);
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setClass(MRJobConfig.COMBINE_CLASS_ATTR, NewReducer.class,
      Object.class);
  TaskContext taskContext = getTaskContext(conf);
  MRCombiner combiner = new MRCombiner(taskContext);
  Writer writer = Mockito.mock(Writer.class);
  combiner.combine(new TezRawKeyValueIteratorTest(), writer);
  long inputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS).getValue();
  long outputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS).getValue();
  assertEquals(6, inputRecords);
  assertEquals(3, outputRecords);
  // verify combiner output keys and values
  verifyKeyAndValues(writer);
}
 

@Ignore
@Test (timeout = 60000)
public void testTaskEventsProcessingSpeed() throws Exception {
  Logger.getRootLogger().setLevel(Level.WARN);
  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  tezconf.setBoolean(TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER, true);
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null,
      null, false, false, 30, 1000);
  tezClient.start();

  final String vAName = "A";
  
  DAG dag = DAG.create("testTaskEventsProcessingSpeed");
  Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 50000);
  dag.addVertex(vA);

  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  mockApp.doSleep = false;
  DAGClient dagClient = tezClient.submitDAG(dag);
  DAGStatus status = dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
  tezClient.stop();
}
 

/**
 * 
 * @throws Exception
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnored() throws Exception {
  URL[] cp = TezClassLoader.getInstance().getURLs();
  StringBuffer buffer = new StringBuffer();
  for (URL url : cp) {
    buffer.append(url.toExternalForm());
    buffer.append(",");
  }
  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, buffer.toString());
  conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  Assert.assertFalse(TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap));
  assertTrue(localizedMap.isEmpty());
}
 

private TezConfiguration createSessionConfig(Path remoteStagingDir) {
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 10);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  return tezConf;
}
 

@Test (timeout=5000)
public void validateSetTezAuxLocalResourcesWithFilesAndFolders() throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  List<String> resources = new ArrayList<String>();
  StringBuilder auxUriStr = new StringBuilder();

  // Create 2 files
  Path topDir = new Path(TEST_ROOT_DIR, "validateauxwithfiles");
  if (localFs.exists(topDir)) {
    localFs.delete(topDir, true);
  }
  localFs.mkdirs(topDir);
  resources.add(createFile(localFs, topDir, "f1.txt").toString());
  auxUriStr.append(localFs.makeQualified(topDir).toString()).append(",");

  Path dir2 = new Path(topDir, "dir2");
  localFs.mkdirs(dir2);
  Path nestedDir = new Path(dir2, "nestedDir");
  localFs.mkdirs(nestedDir);
  createFile(localFs, nestedDir, "nested-f.txt");
  resources.add(createFile(localFs, dir2, "dir2-f.txt").toString());
  auxUriStr.append(localFs.makeQualified(dir2).toString()).append(",");

  Path dir3 = new Path(topDir, "dir3");
  localFs.mkdirs(dir3);
  auxUriStr.append(localFs.makeQualified(dir3).toString()).append(",");

  TezConfiguration conf = new TezConfiguration();
  conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
  conf.set(TezConfiguration.TEZ_AUX_URIS, auxUriStr.toString());
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
  Set<String> resourceNames = localizedMap.keySet();
  Assert.assertEquals(2, resourceNames.size());
  Assert.assertTrue(resourceNames.contains("f1.txt"));
  Assert.assertTrue(resourceNames.contains("dir2-f.txt"));
}
 

@Test(timeout = 60000)
public void testInvalidQueueSubmission() throws Exception {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  YarnClient yarnClient = YarnClient.createYarnClient();
  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");

    String[] inputPaths = new String[1];
    String[] outputPaths = new String[1];
    String inputDirStr = "/tmp/owc-input";
    inputPaths[0] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    String outputDirStr = "/tmp/owc-output";
    outputPaths[0] = outputDirStr;
    int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
        StringUtils.join(",", outputPaths), "2" }, null);
    Assert.assertTrue("Job should have failed", result != 0);
  } catch (TezException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit application"));
  } finally {
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 

private TezConfiguration createConf() {
  TezConfiguration conf = new TezConfiguration();
  conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
  if (useDfs) {
    conf.set("fs.defaultFS", remoteFs.getUri().toString());
  } else {
    conf.set("fs.defaultFS", "file:///");
  }
  conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
  return conf;
}
 

private TezConfiguration createCommonTezLog() throws Exception {
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  
  tezConf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100);
  Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  
  return tezConf;
}
 

@Test (timeout=50000)
public void testTimelineServiceDisabled() throws Exception {
  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      ATSHistoryLoggingService.class.getName());
  tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,false);
  ATSHistoryACLPolicyManager historyACLPolicyManager = ReflectionUtils.createClazzInstance(
      atsHistoryACLManagerClassName);
  historyACLPolicyManager.setConf(tezConf);
  Assert.assertNull(historyACLPolicyManager.timelineClient);
}
 

@Before
public void setup() throws Exception {
  LOG.info("Starting session");
  Path remoteStagingDir =
      remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
          .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, MAX_AM_ATTEMPT);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(
      TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, false);
  tezConf.setBoolean(
      RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED,
      true);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000);
  tezSession = TezClient.create("TestDAGRecovery", tezConf);
  tezSession.start();
}
 

private void startNonSessionClient() throws Exception {
  LOG.info("Starting Client");
  tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 0);
  tezConf
      .setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
  tezClient = TezClient.create("TestExceptionPropagation", tezConf);
  tezClient.start();
}
 

@Private
protected TezClient(String name, TezConfiguration tezConf, boolean isSession,
          @Nullable Map<String, LocalResource> localResources,
          @Nullable Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor) {
  this.clientName = name;
  this.isSession = isSession;
  // Set in conf for local mode AM to figure out whether in session mode or not
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession);
  try {
    InetAddress ip = InetAddress.getLocalHost();
    if (ip != null) {
      tezConf.set(TezConfigurationConstants.TEZ_SUBMIT_HOST, ip.getCanonicalHostName());
      tezConf.set(TezConfigurationConstants.TEZ_SUBMIT_HOST_ADDRESS, ip.getHostAddress());
    }
  } catch (UnknownHostException e) {
    LOG.warn("The host name of the client the tez application was submitted from was unable to be retrieved", e);
  }

  this.ugiMap = new HashMap<>();
  this.amConfig = new AMConfiguration(tezConf, localResources, credentials);
  this.apiVersionInfo = new TezApiVersionInfo();
  this.servicePluginsDescriptor = servicePluginsDescriptor;
  this.maxSubmitDAGRequestSizeThroughIPC = tezConf.getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH,
      CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT) -
      tezConf.getInt(TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES,
      TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT);
  Limits.setConfiguration(tezConf);

  LOG.info("Tez Client Version: " + apiVersionInfo.toString());
}
 

private TezConfiguration createSessionConfig(Path remoteStagingDir) {
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, 10);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());
  tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  return tezConf;
}
 

private void testHashJoinExample(SimpleShutdownCondition shutdownCondition,
    boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception {
  HashJoinExample hashJoinExample = new HashJoinExample();
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.set(TezConfiguration.TEZ_AM_RECOVERY_SERVICE_CLASS,
      RecoveryServiceWithEventHandlingHook.class.getName());
  tezConf.set(
      RecoveryServiceWithEventHandlingHook.AM_RECOVERY_SERVICE_HOOK_CLASS,
      SimpleRecoveryEventHook.class.getName());
  tezConf.set(SimpleRecoveryEventHook.SIMPLE_SHUTDOWN_CONDITION,
      shutdownCondition.serialize());
  tezConf.setBoolean(
      ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
      enableAutoParallelism);
  tezConf.setBoolean(
      RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, false);
  tezConf.setBoolean(
      TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, false);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO;org.apache.tez=DEBUG");

  hashJoinExample.setConf(tezConf);
  Path stagingDirPath = new Path("/tmp/tez-staging-dir");
  Path inPath1 = new Path("/tmp/hashJoin/inPath1");
  Path inPath2 = new Path("/tmp/hashJoin/inPath2");
  Path outPath = new Path("/tmp/hashJoin/outPath");
  remoteFs.delete(outPath, true);
  remoteFs.mkdirs(inPath1);
  remoteFs.mkdirs(inPath2);
  remoteFs.mkdirs(stagingDirPath);

  Set<String> expectedResult = new HashSet<String>();

  FSDataOutputStream out1 = remoteFs.create(new Path(inPath1, "file"));
  FSDataOutputStream out2 = remoteFs.create(new Path(inPath2, "file"));
  BufferedWriter writer1 = new BufferedWriter(new OutputStreamWriter(out1));
  BufferedWriter writer2 = new BufferedWriter(new OutputStreamWriter(out2));
  for (int i = 0; i < 20; i++) {
    String term = "term" + i;
    writer1.write(term);
    writer1.newLine();
    if (i % 2 == 0) {
      writer2.write(term);
      writer2.newLine();
      expectedResult.add(term);
    }
  }
  writer1.close();
  writer2.close();
  out1.close();
  out2.close();

  String[] args = null;
  if (generateSplitInClient) {
    args = new String[]{
        "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "="
            + stagingDirPath.toString(),
        "-generateSplitInClient",
        inPath1.toString(), inPath2.toString(), "1", outPath.toString()};
  } else {
    args = new String[]{
        "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "="
            + stagingDirPath.toString(),
        inPath1.toString(), inPath2.toString(), "1", outPath.toString()};
  }
  assertEquals(0, hashJoinExample.run(args));

  FileStatus[] statuses = remoteFs.listStatus(outPath, new PathFilter() {
    public boolean accept(Path p) {
      String name = p.getName();
      return !name.startsWith("_") && !name.startsWith(".");
    }
  });
  assertEquals(1, statuses.length);
  FSDataInputStream inStream = remoteFs.open(statuses[0].getPath());
  BufferedReader reader = new BufferedReader(new InputStreamReader(inStream));
  String line;
  while ((line = reader.readLine()) != null) {
    assertTrue(expectedResult.remove(line));
  }
  reader.close();
  inStream.close();
  assertEquals(0, expectedResult.size());

  List<HistoryEvent> historyEventsOfAttempt1 = RecoveryParser
      .readRecoveryEvents(tezConf, hashJoinExample.getAppId(), 1);
  HistoryEvent lastEvent = historyEventsOfAttempt1
      .get(historyEventsOfAttempt1.size() - 1);
  assertEquals(shutdownCondition.getEvent().getEventType(),
      lastEvent.getEventType());
  assertTrue(shutdownCondition.match(lastEvent));
}
 

/**
 * In unpartitioned cartesian product, failure fraction should be #unique failure/#consumer that
 * depends on the src task. Here we test a 2x2 cartesian product and let 4th destination task fail.
 * The failure fraction limit is configured to be 0.25. So the failure fraction should be 1/2,
 * not 1/4.
 * @throws Exception
 */
@Test
public void testCartesianProduct() throws Exception {
  Configuration dagConf = new Configuration();
  dagConf.setDouble(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES_FRACTION, 0.25);
  DAG dag = DAG.create("dag");

  Configuration vertexConf = new Configuration();
  vertexConf.setInt(TestProcessor.getVertexConfName(
    TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), 3);
  vertexConf.setInt(TestProcessor.getVertexConfName(
    TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3"), 5);
  UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(vertexConf);
  ProcessorDescriptor processorDescriptor =
    ProcessorDescriptor.create(TestProcessor.class.getName()).setUserPayload(vertexPayload);
  Vertex v1 = Vertex.create("v1", processorDescriptor, 2);
  Vertex v2 = Vertex.create("v2", processorDescriptor, 2);
  Vertex v3 = Vertex.create("v3", processorDescriptor);

  String[] sourceVertices = {"v1", "v2"};
  CartesianProductConfig cartesianProductConfig =
    new CartesianProductConfig(Arrays.asList(sourceVertices));
  TezConfiguration tezConf = new TezConfiguration();
  tezConf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS, 1);
  tezConf.setBoolean(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, false);
  UserPayload cartesianProductPayload =
    cartesianProductConfig.toUserPayload(tezConf);

  v3.setVertexManagerPlugin(
    VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName())
      .setUserPayload(cartesianProductPayload));

  EdgeManagerPluginDescriptor edgeManagerPluginDescriptor =
    EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName())
      .setUserPayload(cartesianProductPayload);

  Configuration inputConf = new Configuration();
  inputConf.setBoolean(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), 3);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), 0);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), 0);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0);
  UserPayload inputPayload = TezUtils.createUserPayloadFromConf(inputConf);
  EdgeProperty edgeProperty =
    EdgeProperty.create(edgeManagerPluginDescriptor, DataMovementType.CUSTOM,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(null),
      TestInput.getInputDesc(inputPayload));
  Edge e1 = Edge.create(v1, v3, edgeProperty);
  Edge e2 = Edge.create(v2, v3, edgeProperty);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(e1).addEdge(e2);

  // run dag
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Test(timeout = 60000)
public void testSimpleSessionExample() throws Exception {
  Path stagingDirPath = new Path("/tmp/owc-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  int numIterations = 2;
  String[] inputPaths = new String[numIterations];
  String[] outputPaths = new String[numIterations];
  Path[] outputDirs = new Path[numIterations];
  for (int i=0; i<numIterations; ++i) {
    String inputDirStr = "/tmp/owc-input-" + i + "/";
    inputPaths[i] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    generateOrderedWordCountInput(inputDir, remoteFs);
    String outputDirStr = "/tmp/owc-output-" + i + "/"; 
    outputPaths[i] = outputDirStr;
    Path outputDir = new Path(outputDirStr);
    outputDirs[i] = outputDir;
  }


  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  YarnClient yarnClient = YarnClient.createYarnClient();

  try {
    
    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();
    
    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    Assert.assertTrue(
        "SimpleSessionExample failed",
        job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
            StringUtils.join(",", outputPaths), "2" }, null) == 0);

    for (int i=0; i<numIterations; ++i) {
      verifyOutput(outputDirs[i], remoteFs);
    }
    
    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;
    
    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }

}