Java Code Examples for org.apache.tez.dag.api.client.DAGStatus#getState()

/**
 * Updates the statistics after a DAG is finished.
 */
public void accumulateStats(TezJob tezJob) throws IOException {
    DAGStatus dagStatus = tezJob.getDAGStatus();
    TezDAGStats tezDAGStats = tezDAGStatsMap.get(tezJob.getName());
    if (dagStatus == null) {
        tezDAGStats.setSuccessful(false);
        tezScriptState.emitJobFailedNotification(tezDAGStats);
        return;
    } else {
        tezDAGStats.accumulateStats(tezJob);
        for(OutputStats output: tezDAGStats.getOutputs()) {
            tezScriptState.emitOutputCompletedNotification(output);
        }
        if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
            tezDAGStats.setSuccessful(true);
            tezScriptState.emitjobFinishedNotification(tezDAGStats);
        } else if (dagStatus.getState() == DAGStatus.State.FAILED) {
            tezDAGStats.setSuccessful(false);
            tezDAGStats.setErrorMsg(tezJob.getDiagnostics());
            tezScriptState.emitJobFailedNotification(tezDAGStats);
        }
        tezScriptState.dagCompletedNotification(tezJob.getName(), tezDAGStats);
    }
}
 

public void notifyUpdate() {
    DAGStatus dagStatus = runningJob.getDAGStatus();
    if (dagStatus != null && dagStatus.getState() == DAGStatus.State.RUNNING) {
        // Emit notification when the job has progressed more than 1%,
        // or every 20 seconds
        int currProgress = Math.round(runningJob.getDAGProgress() * 100f);
        if (currProgress - prevProgress >= 1 || count % 100 == 0) {
            tezScriptState.dagProgressNotification(runningJob.getName(), -1, currProgress);
            tezScriptState.emitProgressUpdatedNotification((currProgress + (100 * processedDAGS))/totalDAGs);
            prevProgress = currProgress;
        }
        count++;
    }
    // TODO: Add new vertex tracking methods to PigTezProgressNotificationListener
    // and emit notifications for individual vertex start, progress and completion
}
 

@Override
public int runDag(DAG dag, boolean printCounters, Logger logger) throws TezException,
    InterruptedException, IOException {
  tezClientInternal.waitTillReady();
  dagClient = tezClientInternal.submitDAG(dag);
  Set<StatusGetOpts> getOpts = new HashSet<StatusGetOpts>();
  if (printCounters) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }

  DAGStatus dagStatus;
  dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);

  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    logger.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;
}
 

public boolean notifyFinishedOrFailed() {
    DAGStatus dagStatus = runningJob.getDAGStatus();
    if (dagStatus == null) {
        return false;
    }
    if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
        Map<Enum, Long> warningAggMap = new HashMap<Enum, Long>();
        DAG dag = runningJob.getDAG();
        for (Vertex v : dag.getVertices()) {
            TezVertexStats tts = tezStats.getVertexStats(dag.getName(), v.getName());
            if (tts == null) {
                continue; //vertex groups
            }
            Map<String, Map<String, Long>> counterGroups = tts.getCounters();
            if (counterGroups == null) {
                log.warn("Counters are not available for vertex " + v.getName() + ". Not computing warning aggregates.");
            } else {
                computeWarningAggregate(counterGroups, warningAggMap);
            }
        }
        if (aggregateWarning) {
            CompilationMessageCollector.logAggregate(warningAggMap, MessageType.Warning, log);
        }
        return true;
    }
    return false;
}
 

boolean run(Configuration conf, String className, String confFilePath) throws Exception {
  this.conf = conf;
  setup();
  
  try {
    tezSession.waitTillReady();
    
    DAG dag = getDAG(className, confFilePath);
    
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      System.out.println("Waiting for dag to complete. Sleeping for 500ms."
          + " DAG name: " + dag.getName()
          + " DAG appId: " + dagClient.getApplicationId()
          + " Current state: " + dagStatus.getState());
      Thread.sleep(500);
      dagStatus = dagClient.getDAGStatus(null);
    }
    
    if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
      return true;
    }
    
  } finally {
    tearDown();
  }
  
  return false;
}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
    throws IOException, TezException, InterruptedException {
  LOG.info("Running IntersectExample");

  UserGroupInformation.setConfiguration(tezConf);

  String streamInputDir = args[0];
  String hashInputDir = args[1];
  int numPartitions = Integer.parseInt(args[2]);
  String outputDir = args[3];

  Path streamInputPath = new Path(streamInputDir);
  Path hashInputPath = new Path(hashInputDir);
  Path outputPath = new Path(outputDir);

  // Verify output path existence
  FileSystem fs = FileSystem.get(tezConf);
  if (fs.exists(outputPath)) {
    System.err.println("Output directory: " + outputDir + " already exists");
    return 3;
  }
  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  DAG dag = createDag(tezConf, streamInputPath, hashInputPath, outputPath, numPartitions);
  setupURIsForCredentials(dag, streamInputPath, hashInputPath, outputPath);

  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;

}
 

/**
 * @param dag           the dag to execute
 * @param printCounters whether to print counters or not
 * @param logger        the logger to use while printing diagnostics
 * @return Zero indicates success, non-zero indicates failure
 * @throws TezException
 * @throws InterruptedException
 * @throws IOException
 */
public int runDag(DAG dag, boolean printCounters, Logger logger) throws TezException,
    InterruptedException, IOException {
  tezClientInternal.waitTillReady();

  CallerContext callerContext = CallerContext.create("TezExamples",
      "Tez Example DAG: " + dag.getName());
  ApplicationId appId = tezClientInternal.getAppMasterApplicationId();
  if (hadoopShim == null) {
    Configuration conf = (getConf() == null ? new Configuration(false) : getConf());
    hadoopShim = new HadoopShimsLoader(conf).getHadoopShim();
  }

  if (appId != null) {
    TezUtilsInternal.setHadoopCallerContext(hadoopShim, appId);
    callerContext.setCallerIdAndType(appId.toString(), "TezExampleApplication");
  }
  dag.setCallerContext(callerContext);

  DAGClient dagClient = tezClientInternal.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (printCounters) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }

  DAGStatus dagStatus;
  dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);

  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    logger.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;
}
 

boolean run(Configuration conf, String className, String confFilePath) throws Exception {
  this.conf = conf;
  setup();
  
  try {
    tezSession.waitTillReady();
    
    DAG dag = getDAG(className, confFilePath);
    
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      System.out.println("Waiting for dag to complete. Sleeping for 500ms."
          + " DAG name: " + dag.getName()
          + " DAG appContext: " + dagClient.getExecutionContext()
          + " Current state: " + dagStatus.getState());
      Thread.sleep(500);
      dagStatus = dagClient.getDAGStatus(null);
    }
    
    if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
      return true;
    }
    
  } finally {
    tearDown();
  }
  
  return false;
}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezClient)
        throws IOException, TezException, InterruptedException {
    LOG.info("Running TopK DataGen");

    UserGroupInformation.setConfiguration(tezConf);

    String outDir = args[0];
    long outDirSize = Long.parseLong(args[1]);

    int numExtraColumns = 0;
    if (args.length > 2) {
        numExtraColumns = Integer.parseInt(args[2]);
    }
    int numTasks = 5;
    if (args.length > 3) {
        numTasks = Integer.parseInt(args[3]);
    }

    Path outPath = new Path(outDir);

    // Verify output path existence
    FileSystem fs = FileSystem.get(tezConf);
    int res = 0;
    res = checkOutputDirectory(fs, outPath);
    if (res != 0) {
        return 3;
    }

    if (numTasks <= 0) {
        System.err.println("NumTasks must be > 0");
        return 4;
    }

    DAG dag = createDag(tezConf, outPath, outDirSize, numExtraColumns, numTasks);

    tezClient.waitTillReady();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
    if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
        return -1;
    }
    return 0;

}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
    throws IOException, TezException, InterruptedException {
  LOG.info("Running IntersectValidate");
  UserGroupInformation.setConfiguration(tezConf);

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
  setupURIsForCredentials(dag, lhsPath, rhsPath);

  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Vlidation successful. The two sides are equivalent");
        return 0;
      }
    }
  }
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  LOG.info("Running JoinValidate");

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);

  tezClient.waitTillReady();
  DAGClient dagClient = tezClient.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (isCountersLog()) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }
  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Validation successful. The two sides are equivalent");
        return 0;
      }
    }
  }

}
 

@Override
public int run(String[] args) throws Exception {
  this.tezConf = new TezConfiguration(getConf());
  String dagName = "pipelinedShuffleTest";
  DAG dag = DAG.create(dagName);

  Vertex m1_Vertex = Vertex.create("mapper1",
      ProcessorDescriptor.create(DataGenerator.class.getName()), 1);

  Vertex m2_Vertex = Vertex.create("mapper2",
      ProcessorDescriptor.create(DataGenerator.class.getName()), 1);

  Vertex reducerVertex = Vertex.create("reducer",
      ProcessorDescriptor.create(SimpleReduceProcessor.class.getName()), 1);

  Edge mapper1_to_reducer = Edge.create(m1_Vertex, reducerVertex,
      OrderedPartitionedKVEdgeConfig
          .newBuilder(Text.class.getName(), Text.class.getName(),
              HashPartitioner.class.getName())
          .setFromConfiguration(tezConf).build().createDefaultEdgeProperty());

  Edge mapper2_to_reducer = Edge.create(m2_Vertex, reducerVertex,
      OrderedPartitionedKVEdgeConfig
          .newBuilder(Text.class.getName(), Text.class.getName(),
              HashPartitioner.class.getName())
          .setFromConfiguration(tezConf).build().createDefaultEdgeProperty());

  dag.addVertex(m1_Vertex);
  dag.addVertex(m2_Vertex);
  dag.addVertex(reducerVertex);

  dag.addEdge(mapper1_to_reducer).addEdge(mapper2_to_reducer);

  TezClient client = TezClient.create(dagName, tezConf);
  client.start();
  client.waitTillReady();

  DAGClient dagClient = client.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  getOpts.add(StatusGetOpts.GET_COUNTERS);

  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);

  System.out.println(dagStatus.getDAGCounters());
  TezCounters counters = dagStatus.getDAGCounters();

  //Ensure that atleast 10 spills were there in this job.
  assertTrue(counters.findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT).getValue() > 10);

  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;
}
 

@Test(timeout=30000)
public void testMultiDAGsOnSession() throws IOException, TezException, InterruptedException {
  int dags = 2;//two dags will be submitted to session
  String[] inputPaths = new String[dags];
  String[] outputPaths =  new String[dags];
  DAGClient[] dagClients = new DAGClient[dags];

  TezConfiguration tezConf = createConf();
  TezClient tezClient = TezClient.create("testMultiDAGOnSession", tezConf, true);
  tezClient.start();

  //create inputs and outputs
  FileSystem fs = FileSystem.get(tezConf);
  for(int i = 0; i < dags; i++) {
    inputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "in-" + i).toString();
    createInputFile(fs, inputPaths[i]);
    outputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "out-" + i).toString();
  }

  //start testing
  try {
    for (int i=0; i<inputPaths.length; ++i) {
      DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], 1,
          false, false, ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session

      tezClient.waitTillReady();
      System.out.println("Running dag number " + i);
      dagClients[i] = tezClient.submitDAG(dag);

      // wait to finish
      DAGStatus dagStatus = dagClients[i].waitForCompletion();
      if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        fail("Iteration " + i + " failed with diagnostics: "
            + dagStatus.getDiagnostics());
      }
      //verify all dags sharing the same execution context
      if(i>0) {
        assertTrue(dagClients[i-1].getExecutionContext().equals(dagClients[i].getExecutionContext()));
      }
    }
  } finally {
    tezClient.stop();
  }
}