Java Code Examples for org.apache.tez.client.TezClient#waitTillReady()

private static SessionInfo createSession(Configuration conf,
        Map<String, LocalResource> requestedAMResources, Credentials creds,
        TezJobConfig tezJobConf) throws TezException, IOException,
        InterruptedException {
    TezConfiguration amConf = MRToTezHelper.getDAGAMConfFromMRConf(conf);
    TezScriptState ss = TezScriptState.get();
    ss.addDAGSettingsToConf(amConf);
    adjustAMConfig(amConf, tezJobConf);
    String jobName = conf.get(PigContext.JOB_NAME, "pig");
    TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds);
    tezClient.start();
    TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus();
    if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) {
        throw new RuntimeException("TezSession has already shutdown");
    }
    tezClient.waitTillReady();
    return new SessionInfo(tezClient, requestedAMResources);
}
 

void runDAGAndVerify(DAG dag, DAGStatus.State finalState,
                     TezClient session) throws Exception {
  session.waitTillReady();
  DAGClient dagClient = session.submitDAG(dag);
  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for dag to complete. Sleeping for 500ms."
        + " DAG name: " + dag.getName()
        + " DAG appId: " + dagClient.getApplicationId()
        + " Current state: " + dagStatus.getState());
    Thread.sleep(100);
    dagStatus = dagClient.getDAGStatus(null);
  }

  Assert.assertEquals(finalState, dagStatus.getState());
}
 

@Test (timeout = 150000)
public void testNonFatalErrors() throws IOException, TezException, InterruptedException {
  String methodName = "testNonFatalErrors";
  TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
  TezClient tezClient = TezClient
      .newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session",
          tezClientConf)
      .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
  try {
    tezClient.start();
    LOG.info("TezSessionStarted for " + methodName);
    tezClient.waitTillReady();
    LOG.info("TezSession ready for submission for " + methodName);


    runAndVerifyForNonFatalErrors(tezClient, SUFFIX_LAUNCHER, EXECUTION_CONTEXT_LAUNCHER_REPORT_NON_FATAL);
    runAndVerifyForNonFatalErrors(tezClient, SUFFIX_TASKCOMM, EXECUTION_CONTEXT_TASKCOMM_REPORT_NON_FATAL);
    runAndVerifyForNonFatalErrors(tezClient, SUFFIX_SCHEDULER, EXECUTION_CONTEXT_SCHEDULER_REPORT_NON_FATAL);

  } finally {
    tezClient.stop();
  }
}
 

void runDAGAndVerify(DAG dag, DAGStatus.State finalState,
                     TezClient session) throws Exception {
  session.waitTillReady();
  DAGClient dagClient = session.submitDAG(dag);
  DAGStatus dagStatus = dagClient.getDAGStatus(null);
  while (!dagStatus.isCompleted()) {
    LOG.info("Waiting for dag to complete. Sleeping for 500ms."
        + " DAG name: " + dag.getName()
        + " DAG appContext: " + dagClient.getExecutionContext()
        + " Current state: " + dagStatus.getState());
    Thread.sleep(100);
    dagStatus = dagClient.getDAGStatus(null);
  }

  Assert.assertEquals(finalState, dagStatus.getState());
}
 

TezClient getTezClient(boolean withTimeline) throws Exception {
  TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
  if (withTimeline) {
    tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, withTimeline);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
        ATSHistoryLoggingService.class.getName());
  } else {
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
        SimpleHistoryLoggingService.class.getName());
  }
  tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);

  TezClient tezClient = TezClient.create("WordCount", tezConf, false);
  tezClient.start();
  tezClient.waitTillReady();
  return tezClient;
}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
    throws IOException, TezException, InterruptedException {
  LOG.info("Running IntersectExample");

  UserGroupInformation.setConfiguration(tezConf);

  String streamInputDir = args[0];
  String hashInputDir = args[1];
  int numPartitions = Integer.parseInt(args[2]);
  String outputDir = args[3];

  Path streamInputPath = new Path(streamInputDir);
  Path hashInputPath = new Path(hashInputDir);
  Path outputPath = new Path(outputDir);

  // Verify output path existence
  FileSystem fs = FileSystem.get(tezConf);
  if (fs.exists(outputPath)) {
    System.err.println("Output directory: " + outputDir + " already exists");
    return 3;
  }
  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  DAG dag = createDag(tezConf, streamInputPath, hashInputPath, outputPath, numPartitions);
  setupURIsForCredentials(dag, streamInputPath, hashInputPath, outputPath);

  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;

}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezClient)
        throws IOException, TezException, InterruptedException {
    LOG.info("Running TopK DataGen");

    UserGroupInformation.setConfiguration(tezConf);

    String outDir = args[0];
    long outDirSize = Long.parseLong(args[1]);

    int numExtraColumns = 0;
    if (args.length > 2) {
        numExtraColumns = Integer.parseInt(args[2]);
    }
    int numTasks = 5;
    if (args.length > 3) {
        numTasks = Integer.parseInt(args[3]);
    }

    Path outPath = new Path(outDir);

    // Verify output path existence
    FileSystem fs = FileSystem.get(tezConf);
    int res = 0;
    res = checkOutputDirectory(fs, outPath);
    if (res != 0) {
        return 3;
    }

    if (numTasks <= 0) {
        System.err.println("NumTasks must be > 0");
        return 4;
    }

    DAG dag = createDag(tezConf, outPath, outDirSize, numExtraColumns, numTasks);

    tezClient.waitTillReady();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
    if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
        return -1;
    }
    return 0;

}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
    throws IOException, TezException, InterruptedException {
  LOG.info("Running IntersectValidate");
  UserGroupInformation.setConfiguration(tezConf);

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
  setupURIsForCredentials(dag, lhsPath, rhsPath);

  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Vlidation successful. The two sides are equivalent");
        return 0;
      }
    }
  }
}
 

private static void waitForTezSessionReady(TezClient tezSession)
  throws IOException, TezException {
  tezSession.waitTillReady();
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  LOG.info("Running JoinValidate");

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);

  tezClient.waitTillReady();
  DAGClient dagClient = tezClient.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (isCountersLog()) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }
  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Validation successful. The two sides are equivalent");
        return 0;
      }
    }
  }

}
 

private static void waitForTezSessionReady(TezClient tezSession)
  throws IOException, TezException, InterruptedException {
  tezSession.waitTillReady();
}
 

@Override
public int run(String[] args) throws Exception {
  this.tezConf = new TezConfiguration(getConf());
  String dagName = "pipelinedShuffleTest";
  DAG dag = DAG.create(dagName);

  Vertex m1_Vertex = Vertex.create("mapper1",
      ProcessorDescriptor.create(DataGenerator.class.getName()), 1);

  Vertex m2_Vertex = Vertex.create("mapper2",
      ProcessorDescriptor.create(DataGenerator.class.getName()), 1);

  Vertex reducerVertex = Vertex.create("reducer",
      ProcessorDescriptor.create(SimpleReduceProcessor.class.getName()), 1);

  Edge mapper1_to_reducer = Edge.create(m1_Vertex, reducerVertex,
      OrderedPartitionedKVEdgeConfig
          .newBuilder(Text.class.getName(), Text.class.getName(),
              HashPartitioner.class.getName())
          .setFromConfiguration(tezConf).build().createDefaultEdgeProperty());

  Edge mapper2_to_reducer = Edge.create(m2_Vertex, reducerVertex,
      OrderedPartitionedKVEdgeConfig
          .newBuilder(Text.class.getName(), Text.class.getName(),
              HashPartitioner.class.getName())
          .setFromConfiguration(tezConf).build().createDefaultEdgeProperty());

  dag.addVertex(m1_Vertex);
  dag.addVertex(m2_Vertex);
  dag.addVertex(reducerVertex);

  dag.addEdge(mapper1_to_reducer).addEdge(mapper2_to_reducer);

  TezClient client = TezClient.create(dagName, tezConf);
  client.start();
  client.waitTillReady();

  DAGClient dagClient = client.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  getOpts.add(StatusGetOpts.GET_COUNTERS);

  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);

  System.out.println(dagStatus.getDAGCounters());
  TezCounters counters = dagStatus.getDAGCounters();

  //Ensure that atleast 10 spills were there in this job.
  assertTrue(counters.findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT).getValue() > 10);

  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;
}
 

@Test(timeout=30000)
public void testMultiDAGsOnSession() throws IOException, TezException, InterruptedException {
  int dags = 2;//two dags will be submitted to session
  String[] inputPaths = new String[dags];
  String[] outputPaths =  new String[dags];
  DAGClient[] dagClients = new DAGClient[dags];

  TezConfiguration tezConf = createConf();
  TezClient tezClient = TezClient.create("testMultiDAGOnSession", tezConf, true);
  tezClient.start();

  //create inputs and outputs
  FileSystem fs = FileSystem.get(tezConf);
  for(int i = 0; i < dags; i++) {
    inputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "in-" + i).toString();
    createInputFile(fs, inputPaths[i]);
    outputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "out-" + i).toString();
  }

  //start testing
  try {
    for (int i=0; i<inputPaths.length; ++i) {
      DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], 1,
          false, false, ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session

      tezClient.waitTillReady();
      System.out.println("Running dag number " + i);
      dagClients[i] = tezClient.submitDAG(dag);

      // wait to finish
      DAGStatus dagStatus = dagClients[i].waitForCompletion();
      if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        fail("Iteration " + i + " failed with diagnostics: "
            + dagStatus.getDiagnostics());
      }
      //verify all dags sharing the same execution context
      if(i>0) {
        assertTrue(dagClients[i-1].getExecutionContext().equals(dagClients[i].getExecutionContext()));
      }
    }
  } finally {
    tezClient.stop();
  }
}