org.apache.tez.dag.api.EdgeProperty.DataSourceType Java Examples

The following examples show how to use org.apache.tez.dag.api.EdgeProperty.DataSourceType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #2
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyCustomEdge() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(EdgeManagerPluginDescriptor.create("emClass"),
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #3
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #4
Source File: SimpleTestDAG.java    From tez with Apache License 2.0 6 votes vote down vote up 
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
Example #5
Source File: SimpleTestDAG3Vertices.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(new Edge(v2, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
Example #6
Source File: SimpleTestDAG.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  return dag;
}
Example #7
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #8
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);

  v1.addDataSink("v2", DataSinkDescriptor.create(null, null, null));

  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #9
Source File: TestAMRecovery.java    From tez with Apache License 2.0 6 votes vote down vote up 
/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}
Example #10
Source File: SimpleTestDAG3Vertices.java    From tez with Apache License 2.0 6 votes vote down vote up 
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
Example #11
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testSelfCycle(){
  IllegalStateException ex=null;
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v1,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addEdge(e1);
  try{
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNotNull(ex);
  System.out.println(ex.getMessage());
  Assert.assertTrue(ex.getMessage().startsWith("DAG contains a self-cycle"));
}
Example #12
Source File: TestEdge.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testInvalidConsumerNumber() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1,1,0,1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1),
        new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1",
            TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
    edge.sendTezEventToSourceTasks(ireEvent);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("ConsumerTaskNum must be positive"));
  }
}
Example #13
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDAGWithSplitsOnClient() {
  DAG dag = DAG.create("testDag");

  // Mimic map which has a data source and shards set when splits are generated in the client
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  v1.addDataSource("input", DataSourceDescriptor.create(
      InputDescriptor.create(dummyInputClassName), null, 10, null, null, null));
  dag.addVertex(v1);

  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create(dummyProcessorClassName));
  dag.addVertex(v2);

  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(
      DataMovementType.ONE_TO_ONE,
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(dummyOutputClassName),
      InputDescriptor.create(dummyInputClassName))));

  dag.verify();
}
Example #14
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testDAGInvalidParallelism3() {
  DAG dag = DAG.create("testDAG");
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  dag.addVertex(v1);
  try {
    dag.verify();
    Assert.fail();
  } catch (Exception e) {
    Assert.assertEquals(
        "v1 has -1 tasks but does not have input initializers, 1-1 uninited sources or custom vertex manager to set it at runtime",
        e.getMessage());
  }

  Vertex v0 = Vertex.create("v0", ProcessorDescriptor.create(dummyProcessorClassName));
  DataSourceDescriptor dsDesc = DataSourceDescriptor.create(InputDescriptor.create(dummyInputClassName),
      InputInitializerDescriptor.create(dummyInputInitClassName), null);
  v0.addDataSource("input", dsDesc);
  dag.addVertex(v0);
  dag.addEdge(Edge.create(v0, v1, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
      DataSourceType.PERSISTED,SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(dummyOutputClassName),
      InputDescriptor.create(dummyInputClassName))));
  dag.verify();
}
Example #15
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyCustomEdge() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(new EdgeManagerDescriptor("emClass"),
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #16
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #17
Source File: TestEdge.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testInvalidPhysicalInputCount() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(-1,1,1,1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    edge.getDestinationSpec(0);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("PhysicalInputCount should not be negative"));
  }
}
Example #18
Source File: TestMockDAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up 
private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) {
  DAG dag = DAG.create(dagName);
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1);
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1);
  Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1);
  VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
  DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null);
  uv12.addDataSink("uv12Out", uv12DataSink);
  DataSinkDescriptor v3DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null);
  v3.addDataSink("v3Out", v3DataSink);

  GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create("dummy output class"),
      InputDescriptor.create("dummy input class")), InputDescriptor
      .create("merge.class"));
  dag.addVertex(v1)
    .addVertex(v2)
    .addVertex(v3)
    .addEdge(e1);
  return dag;
}
Example #19
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerify4() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #20
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      -1, dummyTaskResource);
  DataSourceDescriptor dsDesc = DataSourceDescriptor.create(InputDescriptor.create(dummyInputClassName),
      InputInitializerDescriptor.create(dummyInputInitClassName), null);
  v1.addDataSource("input_1", dsDesc);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}
Example #21
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #22
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify3() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #23
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify4() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #24
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.BROADCAST,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #25
Source File: TestDAGVerify.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerify3() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #26
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addInput("v1", new InputDescriptor(), null);
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #27
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addOutput("v2", new OutputDescriptor());
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example #28
Source File: TestExceptionPropagation.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * create a DAG with 2 vertices (v1 --> v2), set payload on Input/Output/Processor/VertexManagerPlugin to
 * control where throw exception
 * 
 * @param exLocation
 * @return
 * @throws IOException
 */
private DAG createDAG(ExceptionLocation exLocation) throws IOException {
  DAG dag = DAG.create("dag_" + exLocation.name());
  UserPayload payload =
      UserPayload.create(ByteBuffer.wrap(exLocation.name().getBytes()));
  Vertex v1 =
      Vertex.create("v1", ProcessorWithException.getProcDesc(payload), 1);
  InputDescriptor inputDesc = InputWithException.getInputDesc(payload);
  InputInitializerDescriptor iiDesc =
      InputInitializerWithException.getIIDesc(payload);
  v1.addDataSource("input",
      DataSourceDescriptor.create(inputDesc, iiDesc, null));
  v1.setVertexManagerPlugin(RootInputVertexManagerWithException
      .getVMDesc(exLocation));

  Vertex v2 = 
      Vertex.create("v2", DoNothingProcessor.getProcDesc(), 1);
  v2.addDataSource("input2",
      DataSourceDescriptor.create(InputDescriptor.create(NoOpInput.class.getName()),
        InputInitializerWithException2.getIIDesc(payload), null));

  dag.addVertex(v1)
    .addVertex(v2);
  if (exLocation.name().startsWith("EM_")) {
    dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(
        EdgeManagerPluginDescriptor.create(CustomEdgeManager.class.getName())
          .setUserPayload(payload),
        DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
        OutputWithException.getOutputDesc(payload), InputWithException.getInputDesc(payload))));
  } else {
    // set Customized VertexManager here, it can't been used for CustomEdge
    v2.setVertexManagerPlugin(InputReadyVertexManagerWithException.getVMDesc(exLocation));
    dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
        DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
        OutputWithException.getOutputDesc(payload), InputWithException.getInputDesc(payload))));
  }

  return dag;
}
Example #29
Source File: TestDAGVerify.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
// v1 (-1) -> v2 (known) -> v3 (-1)
public void testVerifyOneToOneIncorrectParallelism1() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = Vertex.create("v3",
      ProcessorDescriptor.create("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = Edge.create(v1, v3,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  Edge e2 = Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  try {
    dag.verify();
    Assert.assertTrue(false);
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains(
        "1-1 Edge. Destination vertex parallelism must match source vertex"));
  }
}
Example #30
Source File: TestFaultTolerance.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test (timeout=60000)
public void testBasicSuccessBroadcast() throws Exception {
  DAG dag = new DAG("testBasicSuccessBroadcast");
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(null), 
          TestInput.getInputDesc(null))));
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}