Java Code Examples for org.apache.tez.dag.api.TezConfiguration#TEZ_AM_STAGING_DIR

The following examples show how to use org.apache.tez.dag.api.TezConfiguration#TEZ_AM_STAGING_DIR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestTezJobs.java From incubator-tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testIntersectExample() throws Exception {
  IntersectExample intersectExample = new IntersectExample();
  intersectExample.setConf(new Configuration(mrrTezCluster.getConfig()));
  Path stagingDirPath = new Path("/tmp/tez-staging-dir");
  Path inPath1 = new Path("/tmp/inPath1");
  Path inPath2 = new Path("/tmp/inPath2");
  Path outPath = new Path("/tmp/outPath");
  remoteFs.mkdirs(inPath1);
  remoteFs.mkdirs(inPath2);
  remoteFs.mkdirs(stagingDirPath);

  Set<String> expectedResult = new HashSet<String>();

  FSDataOutputStream out1 = remoteFs.create(new Path(inPath1, "file"));
  FSDataOutputStream out2 = remoteFs.create(new Path(inPath2, "file"));
  BufferedWriter writer1 = new BufferedWriter(new OutputStreamWriter(out1));
  BufferedWriter writer2 = new BufferedWriter(new OutputStreamWriter(out2));
  for (int i = 0; i < 20; i++) {
    String term = "term" + i;
    writer1.write(term);
    writer1.newLine();
    if (i % 2 == 0) {
      writer2.write(term);
      writer2.newLine();
      expectedResult.add(term);
    }
  }
  writer1.close();
  writer2.close();
  out1.close();
  out2.close();

  String[] args = new String[] {
      "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(),
      inPath1.toString(), inPath2.toString(), "1", outPath.toString() };
  assertEquals(0, intersectExample.run(args));

  FileStatus[] statuses = remoteFs.listStatus(outPath, new PathFilter() {
    public boolean accept(Path p) {
      String name = p.getName();
      return !name.startsWith("_") && !name.startsWith(".");
    }
  });
  assertEquals(1, statuses.length);
  FSDataInputStream inStream = remoteFs.open(statuses[0].getPath());
  BufferedReader reader = new BufferedReader(new InputStreamReader(inStream));
  String line = null;
  while ((line = reader.readLine()) != null) {
    assertTrue(expectedResult.remove(line));
  }
  reader.close();
  inStream.close();
  assertEquals(0, expectedResult.size());
}

Example 2

Source File: TestRecovery.java From tez with Apache License 2.0

4 votes

private void testHashJoinExample(SimpleShutdownCondition shutdownCondition,
    boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception {
  HashJoinExample hashJoinExample = new HashJoinExample();
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
  tezConf.set(TezConfiguration.TEZ_AM_RECOVERY_SERVICE_CLASS,
      RecoveryServiceWithEventHandlingHook.class.getName());
  tezConf.set(
      RecoveryServiceWithEventHandlingHook.AM_RECOVERY_SERVICE_HOOK_CLASS,
      SimpleRecoveryEventHook.class.getName());
  tezConf.set(SimpleRecoveryEventHook.SIMPLE_SHUTDOWN_CONDITION,
      shutdownCondition.serialize());
  tezConf.setBoolean(
      ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
      enableAutoParallelism);
  tezConf.setBoolean(
      RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, false);
  tezConf.setBoolean(
      TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, false);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
  tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000);
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO;org.apache.tez=DEBUG");

  hashJoinExample.setConf(tezConf);
  Path stagingDirPath = new Path("/tmp/tez-staging-dir");
  Path inPath1 = new Path("/tmp/hashJoin/inPath1");
  Path inPath2 = new Path("/tmp/hashJoin/inPath2");
  Path outPath = new Path("/tmp/hashJoin/outPath");
  remoteFs.delete(outPath, true);
  remoteFs.mkdirs(inPath1);
  remoteFs.mkdirs(inPath2);
  remoteFs.mkdirs(stagingDirPath);

  Set<String> expectedResult = new HashSet<String>();

  FSDataOutputStream out1 = remoteFs.create(new Path(inPath1, "file"));
  FSDataOutputStream out2 = remoteFs.create(new Path(inPath2, "file"));
  BufferedWriter writer1 = new BufferedWriter(new OutputStreamWriter(out1));
  BufferedWriter writer2 = new BufferedWriter(new OutputStreamWriter(out2));
  for (int i = 0; i < 20; i++) {
    String term = "term" + i;
    writer1.write(term);
    writer1.newLine();
    if (i % 2 == 0) {
      writer2.write(term);
      writer2.newLine();
      expectedResult.add(term);
    }
  }
  writer1.close();
  writer2.close();
  out1.close();
  out2.close();

  String[] args = null;
  if (generateSplitInClient) {
    args = new String[]{
        "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "="
            + stagingDirPath.toString(),
        "-generateSplitInClient",
        inPath1.toString(), inPath2.toString(), "1", outPath.toString()};
  } else {
    args = new String[]{
        "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "="
            + stagingDirPath.toString(),
        inPath1.toString(), inPath2.toString(), "1", outPath.toString()};
  }
  assertEquals(0, hashJoinExample.run(args));

  FileStatus[] statuses = remoteFs.listStatus(outPath, new PathFilter() {
    public boolean accept(Path p) {
      String name = p.getName();
      return !name.startsWith("_") && !name.startsWith(".");
    }
  });
  assertEquals(1, statuses.length);
  FSDataInputStream inStream = remoteFs.open(statuses[0].getPath());
  BufferedReader reader = new BufferedReader(new InputStreamReader(inStream));
  String line;
  while ((line = reader.readLine()) != null) {
    assertTrue(expectedResult.remove(line));
  }
  reader.close();
  inStream.close();
  assertEquals(0, expectedResult.size());

  List<HistoryEvent> historyEventsOfAttempt1 = RecoveryParser
      .readRecoveryEvents(tezConf, hashJoinExample.getAppId(), 1);
  HistoryEvent lastEvent = historyEventsOfAttempt1
      .get(historyEventsOfAttempt1.size() - 1);
  assertEquals(shutdownCondition.getEvent().getEventType(),
      lastEvent.getEventType());
  assertTrue(shutdownCondition.match(lastEvent));
}

Example 3