org.apache.tez.dag.api.EdgeProperty.DataMovementType Java Examples

@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify3() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is
 * not re-run. Task 1 is re-run. (SCATTER_GATHER)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexPartiallyFinished_ScatterGather() throws Exception {
  DAG dag =
      createDAG("VertexPartiallyFinished_ScatterGather", ControlledShuffleVertexManager.class,
          DataMovementType.SCATTER_GATHER, true);
  TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
  assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());

  List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
  List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
  printHistoryEvents(historyEvents1, 1);
  printHistoryEvents(historyEvents1, 2);
  // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
  // attempt 1
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
  assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());

  // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
  // finished in attempt 2
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());

}
 

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is also done. History flush happens. AM dies. Once AM is recovered, task 0
 * and Task 1 is not re-run. (ONE_TO_ONE)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexCompletelyFinished_One2One() throws Exception {
  DAG dag =
      createDAG("VertexCompletelyFinished_One2One", ControlledInputReadyVertexManager.class,
          DataMovementType.ONE_TO_ONE, false);
  TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
  assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());

  List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
  List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
  printHistoryEvents(historyEvents1, 1);
  printHistoryEvents(historyEvents1, 2);
  // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
  // attempt 1
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());

  // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
  // finished in attempt 2
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());

}
 

@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) {
  DAG dag = DAG.create(dagName);
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1);
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1);
  Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1);
  VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
  DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null);
  uv12.addDataSink("uv12Out", uv12DataSink);
  DataSinkDescriptor v3DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null);
  v3.addDataSink("v3Out", v3DataSink);

  GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create("dummy output class"),
      InputDescriptor.create("dummy input class")), InputDescriptor
      .create("merge.class"));
  dag.addVertex(v1)
    .addVertex(v2)
    .addVertex(v3)
    .addEdge(e1);
  return dag;
}
 

/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

@Override
public synchronized void onSourceTaskCompleted(TaskAttemptIdentifier attempt) {
  String srcVertexName = attempt.getTaskIdentifier().getVertexIdentifier().getName();
  int srcTaskId = attempt.getTaskIdentifier().getIdentifier();
  SourceVertexInfo srcInfo = srcVertexInfo.get(srcVertexName);
  if (srcInfo.vertexIsConfigured) {
    Preconditions.checkState(srcTaskId < srcInfo.numTasks,
        "Received completion for srcTaskId " + srcTaskId + " but Vertex: " + srcVertexName +
        " has only " + srcInfo.numTasks + " tasks");
  }
  //handle duplicate events and count task completions from all source vertices
  BitSet completedSourceTasks = srcInfo.finishedTaskSet;
  // duplicate notifications tracking
  if (!completedSourceTasks.get(srcTaskId)) {
    completedSourceTasks.set(srcTaskId);
    // source task has completed
    if (srcInfo.edgeProperty.getDataMovementType() == DataMovementType.SCATTER_GATHER) {
      numBipartiteSourceTasksCompleted++;
    }
  }
  processPendingTasks(attempt);
}
 

private void handleVertexStateUpdate(VertexStateUpdate stateUpdate) {
  Preconditions.checkArgument(stateUpdate.getVertexState() == VertexState.CONFIGURED,
      "Received incorrect state notification : " + stateUpdate.getVertexState() + " for vertex: "
          + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
  Preconditions.checkArgument(srcVertexInfo.containsKey(stateUpdate.getVertexName()),
      "Received incorrect vertex notification : " + stateUpdate.getVertexState() + " for vertex: "
          + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
  SourceVertexInfo vInfo = srcVertexInfo.get(stateUpdate.getVertexName());
  Preconditions.checkState(vInfo.vertexIsConfigured == false);
  vInfo.vertexIsConfigured = true;
  vInfo.numTasks = getContext().getVertexNumTasks(stateUpdate.getVertexName());
  if (vInfo.edgeProperty.getDataMovementType() == DataMovementType.SCATTER_GATHER) {
    totalNumBipartiteSourceTasks += vInfo.numTasks;
  }
  LOG.info("Received configured notification : {}" + " for vertex: {} in" +
      " vertex: {}" + " numBipartiteSourceTasks: {}",
      stateUpdate.getVertexState(), stateUpdate.getVertexName(),
      getContext().getVertexName(), totalNumBipartiteSourceTasks);
  processPendingTasks(null);
}
 

@Test
public void testDAGWithSplitsOnClient() {
  DAG dag = DAG.create("testDag");

  // Mimic map which has a data source and shards set when splits are generated in the client
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  v1.addDataSource("input", DataSourceDescriptor.create(
      InputDescriptor.create(dummyInputClassName), null, 10, null, null, null));
  dag.addVertex(v1);

  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create(dummyProcessorClassName));
  dag.addVertex(v2);

  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(
      DataMovementType.ONE_TO_ONE,
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(dummyOutputClassName),
      InputDescriptor.create(dummyInputClassName))));

  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);

  v2.addDataSource("v1", DataSourceDescriptor.create(null, null, null));

  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

private GroupInputEdge newGroupInputEdge(TezOperator fromOp,
        TezOperator toOp, VertexGroup from, Vertex to) throws IOException {

    EdgeProperty edgeProperty = newEdge(fromOp, toOp);

    String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName();

    // In case of SCATTER_GATHER and UnorderedKVInput it will still be
    // ConcatenatedMergedKeyValueInput
    if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER)
            && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) {
        groupInputClass = OrderedGroupedMergedKVInput.class.getName();
    }

    return GroupInputEdge.create(from, to, edgeProperty,
            InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())
                .setHistoryText(edgeProperty.getEdgeDestination().getHistoryText()));
}
 

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is
 * not re-run. Task 1 is re-run. (ONE_TO_ONE)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexPartialFinished_One2One() throws Exception {
  DAG dag =
      createDAG("VertexPartialFinished_One2One", ControlledInputReadyVertexManager.class,
          DataMovementType.ONE_TO_ONE, true);
  TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
  assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());

  List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
  List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
  printHistoryEvents(historyEvents1, 1);
  printHistoryEvents(historyEvents1, 2);
  // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
  // attempt 1
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
  assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());

  // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
  // finished in attempt 2
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());

}
 

@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

protected static void addDAGVerticesAndEdges() {
    ThreeLevelsFailingDAG.addDAGVerticesAndEdges();
    l4v1 = new Vertex("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource);
    dag.addVertex(l4v1);
    addEdge(l3v1, l4v1, DataMovementType.SCATTER_GATHER);
    addEdge(l3v2, l4v1, DataMovementType.SCATTER_GATHER);
    l5v1 = new Vertex("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource);
    dag.addVertex(l5v1);
    addEdge(l4v1, l5v1, DataMovementType.SCATTER_GATHER);
    l5v2 = new Vertex("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l5v2);
    addEdge(l4v1, l5v2, DataMovementType.SCATTER_GATHER);
    l5v3 = new Vertex("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource);
    dag.addVertex(l5v3);
    addEdge(l4v1, l5v3, DataMovementType.SCATTER_GATHER);
    l6v1 = new Vertex("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l6v1);
    addEdge(l5v1, l6v1, DataMovementType.SCATTER_GATHER);
    addEdge(l5v2, l6v1, DataMovementType.SCATTER_GATHER);
    addEdge(l5v3, l6v1, DataMovementType.SCATTER_GATHER);
}
 

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is
 * not re-run. Task 1 is re-run. (Broadcast)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexPartiallyFinished_Broadcast() throws Exception {
  DAG dag =
      createDAG("VertexPartiallyFinished_Broadcast", ControlledImmediateStartVertexManager.class,
          DataMovementType.BROADCAST, true);
  TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
  assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());

  List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
  List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
  printHistoryEvents(historyEvents1, 1);
  printHistoryEvents(historyEvents1, 2);
  // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
  // attempt 1
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
  assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());

  // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
  // finished in attempt 2
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(new Edge(v2, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
 

protected static void addDAGVerticesAndEdges() {
    ThreeLevelsFailingDAG.addDAGVerticesAndEdges();
    l4v1 = Vertex.create("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource);
    dag.addVertex(l4v1);
    addEdge(l3v1, l4v1, DataMovementType.SCATTER_GATHER);
    addEdge(l3v2, l4v1, DataMovementType.SCATTER_GATHER);
    l5v1 = Vertex.create("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource);
    dag.addVertex(l5v1);
    addEdge(l4v1, l5v1, DataMovementType.SCATTER_GATHER);
    l5v2 = Vertex.create("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l5v2);
    addEdge(l4v1, l5v2, DataMovementType.SCATTER_GATHER);
    l5v3 = Vertex.create("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource);
    dag.addVertex(l5v3);
    addEdge(l4v1, l5v3, DataMovementType.SCATTER_GATHER);
    l6v1 = Vertex.create("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l6v1);
    addEdge(l5v1, l6v1, DataMovementType.SCATTER_GATHER);
    addEdge(l5v2, l6v1, DataMovementType.SCATTER_GATHER);
    addEdge(l5v3, l6v1, DataMovementType.SCATTER_GATHER);
}
 

@Test(timeout = 5000)
public void testSelfCycle(){
  IllegalStateException ex=null;
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v1,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addEdge(e1);
  try{
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNotNull(ex);
  System.out.println(ex.getMessage());
  Assert.assertTrue(ex.getMessage().startsWith("DAG contains a self-cycle"));
}
 

@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      -1, dummyTaskResource);
  DataSourceDescriptor dsDesc = DataSourceDescriptor.create(InputDescriptor.create(dummyInputClassName),
      InputInitializerDescriptor.create(dummyInputInitClassName), null);
  v1.addDataSource("input_1", dsDesc);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);

  v1.addDataSink("v2", DataSinkDescriptor.create(null, null, null));

  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addOutput("v2", new OutputDescriptor());
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addInput("v1", new InputDescriptor(), null);
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.BROADCAST,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test//(timeout = 5000)
public void testDAGInvalidParallelism4() {
  DAG dag = DAG.create("testDAG");
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  dag.addVertex(v1);
  try {
    dag.verify();
    Assert.fail();
  } catch (Exception e) {
    Assert.assertEquals(
        "v1 has -1 tasks but does not have input initializers, 1-1 uninited sources or custom vertex manager to set it at runtime",
        e.getMessage());
  }

  Vertex v0 = Vertex.create("v2", ProcessorDescriptor.create(dummyProcessorClassName));
  v0.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(dummyVMPluginClassName));
  dag.addVertex(v0);
  dag.addEdge(Edge.create(v0, v1, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
      DataSourceType.PERSISTED,SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(dummyOutputClassName),
      InputDescriptor.create(dummyInputClassName))));
  dag.verify();
}
 

@Ignore
@Test (timeout = 600000)
public void testMemoryScatterGather() throws Exception {
  DAG dag = DAG.create("testMemoryScatterGather");
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks);
  Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), numTasks);
  dag.addVertex(vA)
      .addVertex(vB)
      .addEdge(
          Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER,
              DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
              OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
  testMemory(dag, true);
}
 

@Test(timeout = 5000)
public void BinaryOutput() {
  IllegalStateException ex = null;
  try {
    Vertex v1 = Vertex.create("v1",
        ProcessorDescriptor.create("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v2 = Vertex.create("v2",
        ProcessorDescriptor.create("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v3 = Vertex.create("v3",
        ProcessorDescriptor.create("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Edge e1 = Edge.create(v1, v2,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            OutputDescriptor.create("dummy output class"),
            InputDescriptor.create("dummy input class")));
    Edge e2 = Edge.create(v1, v3,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            OutputDescriptor.create("dummy output class"),
            InputDescriptor.create("dummy input class")));
    DAG dag = DAG.create("testDag");
    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    dag.addEdge(e2);
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNull(ex);
}