Java Code Examples for org.apache.tez.dag.api.EdgeProperty.DataSourceType#PERSISTED

The following examples show how to use org.apache.tez.dag.api.EdgeProperty.DataSourceType#PERSISTED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addOutput("v2", new OutputDescriptor());
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example 2
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addInput("v1", new InputDescriptor(), null);
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example 3
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example 4
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example 5
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
Example 6
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
public void testVerifyOneToOneInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v2.getParallelism());
  Assert.assertEquals(dummyTaskCount, v3.getParallelism());
}
Example 7
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
// v1 (-1) -> v2 (known) -> v3 (-1)
public void testVerifyOneToOneIncorrectParallelism1() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  try {
    dag.verify();
    Assert.assertTrue(false);
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains(
        "1-1 Edge. Destination vertex parallelism must match source vertex"));
  }
}
Example 8
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
public void BinaryInputAllowed() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("ReduceProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
}
Example 9
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
// The test checks resiliency to ordering of the vertices/edges
public void testVerifyOneToOneInferParallelismReverseOrder() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v3);
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v2.getParallelism());
  Assert.assertEquals(dummyTaskCount, v3.getParallelism());
}
Example 10
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static DataSourceType convertFromDAGPlan(PlanEdgeDataSourceType sourceType){
  switch(sourceType){
    case PERSISTED : return DataSourceType.PERSISTED;
    case PERSISTED_RELIABLE : return DataSourceType.PERSISTED_RELIABLE;
    case EPHEMERAL :  return DataSourceType.EPHEMERAL;
    default : throw new RuntimeException("unknown 'dataSourceType': " + sourceType);
  }
}
Example 11
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
public void BinaryOutput() {
  IllegalStateException ex = null;
  try {
    Vertex v1 = new Vertex("v1",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v2 = new Vertex("v2",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v3 = new Vertex("v3",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Edge e1 = new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")));
    Edge e2 = new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")));
    DAG dag = new DAG("testDag");
    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    dag.addEdge(e2);
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNull(ex);
}
Example 12
Source File: TestDAGPlan.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 5000)
public void testEdgeManagerSerde() {
  DAG dag = new DAG("testDag");
  ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1")
      .setUserPayload("processor1Bytes".getBytes());
  ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2")
      .setUserPayload("processor2Bytes".getBytes());
  Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = new InputDescriptor("input").setUserPayload("inputBytes"
      .getBytes());
  OutputDescriptor outputDescriptor = new OutputDescriptor("output").setUserPayload("outputBytes"
      .getBytes());
  Edge edge = new Edge(v1, v2, new EdgeProperty(
      new EdgeManagerDescriptor("emClass").setUserPayload("emPayload".getBytes()),
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration());

  EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto
      .getEdgeList().get(0));

  EdgeManagerDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
  Assert.assertNotNull(emDesc);
  Assert.assertEquals("emClass", emDesc.getClassName());
  Assert.assertTrue(Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload()));
}
Example 13
Source File: DagTypeConverters.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
public static DataSourceType convertFromDAGPlan(PlanEdgeDataSourceType sourceType){
  switch(sourceType){
    case PERSISTED : return DataSourceType.PERSISTED;
    case PERSISTED_RELIABLE : return DataSourceType.PERSISTED_RELIABLE;
    case EPHEMERAL :  return DataSourceType.EPHEMERAL;
    default : throw new RuntimeException("unknown 'dataSourceType': " + sourceType);
  }
}
Example 14
Source File: TestDAGPlan.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test (timeout=5000)
public void testCredentialsSerde() {
  DAG dag = new DAG("testDag");
  ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
      setUserPayload("processor1Bytes".getBytes());
  ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
      setUserPayload("processor2Bytes".getBytes());
  Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = new InputDescriptor("input").
      setUserPayload("inputBytes".getBytes());
  OutputDescriptor outputDescriptor = new OutputDescriptor("output").
      setUserPayload("outputBytes".getBytes());
  Edge edge = new Edge(v1, v2, new EdgeProperty(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  Credentials dagCredentials = new Credentials();
  Token<TokenIdentifier> token1 = new Token<TokenIdentifier>();
  Token<TokenIdentifier> token2 = new Token<TokenIdentifier>();
  dagCredentials.addToken(new Text("Token1"), token1);
  dagCredentials.addToken(new Text("Token2"), token2);
  
  dag.setCredentials(dagCredentials);

  DAGPlan dagProto = dag.createDag(new TezConfiguration());

  assertTrue(dagProto.hasCredentialsBinary());
  
  Credentials fetchedCredentials = DagTypeConverters.convertByteStringToCredentials(dagProto
      .getCredentialsBinary());
  
  assertEquals(2, fetchedCredentials.numberOfTokens());
  assertNotNull(fetchedCredentials.getToken(new Text("Token1")));
  assertNotNull(fetchedCredentials.getToken(new Text("Token2")));
}
Example 15
Source File: TestDAGPlan.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void userVertexOrderingIsMaintained() {
  DAG dag = new DAG("testDag");
  ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
      setUserPayload("processor1Bytes".getBytes());
  ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
      setUserPayload("processor2Bytes".getBytes());
  ProcessorDescriptor pd3 = new ProcessorDescriptor("processor3").
      setUserPayload("processor3Bytes".getBytes());
  Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
  Vertex v3 = new Vertex("v3", pd3, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v3.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = new InputDescriptor("input").
      setUserPayload("inputBytes".getBytes());
  OutputDescriptor outputDescriptor = new OutputDescriptor("output").
      setUserPayload("outputBytes".getBytes());
  Edge edge = new Edge(v1, v2, new EdgeProperty(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge).addVertex(v3);

  DAGPlan dagProto = dag.createDag(new TezConfiguration());

  assertEquals(3, dagProto.getVertexCount());
  assertEquals(1, dagProto.getEdgeCount());

  VertexPlan v1Proto = dagProto.getVertex(0);
  VertexPlan v2Proto = dagProto.getVertex(1);
  VertexPlan v3Proto = dagProto.getVertex(2);
  EdgePlan edgeProto = dagProto.getEdge(0);

  assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());

  assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());

  assertEquals("processor3Bytes", new String(v3Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor3", v3Proto.getProcessorDescriptor().getClassName());

  assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
      .getUserPayload().toByteArray()));
  assertEquals("input", edgeProto.getEdgeDestination().getClassName());

  assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
      .getUserPayload().toByteArray()));
  assertEquals("output", edgeProto.getEdgeSource().getClassName());

  EdgeProperty edgeProperty = DagTypeConverters
      .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));

  byte[] ib = edgeProperty.getEdgeDestination().getUserPayload();
  assertEquals("inputBytes", new String(ib));
  assertEquals("input", edgeProperty.getEdgeDestination().getClassName());

  byte[] ob = edgeProperty.getEdgeSource().getUserPayload();
  assertEquals("outputBytes", new String(ob));
  assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}
Example 16
Source File: TestDAGPlan.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void testUserPayloadSerde() {
  DAG dag = new DAG("testDag");
  ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
      setUserPayload("processor1Bytes".getBytes());
  ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
      setUserPayload("processor2Bytes".getBytes());
  Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = new InputDescriptor("input").
      setUserPayload("inputBytes".getBytes());
  OutputDescriptor outputDescriptor = new OutputDescriptor("output").
      setUserPayload("outputBytes".getBytes());
  Edge edge = new Edge(v1, v2, new EdgeProperty(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration());

  assertEquals(2, dagProto.getVertexCount());
  assertEquals(1, dagProto.getEdgeCount());

  VertexPlan v1Proto = dagProto.getVertex(0);
  VertexPlan v2Proto = dagProto.getVertex(1);
  EdgePlan edgeProto = dagProto.getEdge(0);

  assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());

  assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());

  assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
      .getUserPayload().toByteArray()));
  assertEquals("input", edgeProto.getEdgeDestination().getClassName());

  assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
      .getUserPayload().toByteArray()));
  assertEquals("output", edgeProto.getEdgeSource().getClassName());

  EdgeProperty edgeProperty = DagTypeConverters
      .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));

  byte[] ib = edgeProperty.getEdgeDestination().getUserPayload();
  assertEquals("inputBytes", new String(ib));
  assertEquals("input", edgeProperty.getEdgeDestination().getClassName());

  byte[] ob = edgeProperty.getEdgeSource().getUserPayload();
  assertEquals("outputBytes", new String(ob));
  assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}
Example 17
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
// v1 (-1) -> v3 (-1), v2 (known) -> v3 (-1)
// order of edges should not matter
public void testVerifyOneToOneIncorrectParallelism2() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e3 = new Edge(v3, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.addEdge(e3);
  try {
    dag.verify();
    Assert.assertTrue(false);
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains(
        "1-1 Edge. Destination vertex parallelism must match source vertex"));
  }
}
Example 18
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void testVertexGroup() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v5 = new Vertex("v5",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  
  DAG dag = new DAG("testDag");
  String groupName1 = "uv12";
  VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
  OutputDescriptor outDesc = new OutputDescriptor();
  uv12.addOutput("uvOut", outDesc, null);
  
  String groupName2 = "uv23";
  VertexGroup uv23 = dag.createVertexGroup(groupName2, v2, v3);
  
  GroupInputEdge e1 = new GroupInputEdge(uv12, v4,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  GroupInputEdge e2 = new GroupInputEdge(uv23, v5,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addVertex(v5);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();

  // for the first Group v1 and v2 should get connected to v4 and also have 1 output
  // for the second Group v2 and v3 should get connected to v5
  // the Group place holders should disappear
  Assert.assertNull(dag.getVertex(uv12.getGroupName()));
  Assert.assertNull(dag.getVertex(uv23.getGroupName()));
  Assert.assertFalse(dag.edges.contains(e1));
  Assert.assertFalse(dag.edges.contains(e2));
  Assert.assertEquals(1, v1.getOutputs().size());
  Assert.assertEquals(1, v2.getOutputs().size());
  Assert.assertEquals(outDesc, v1.getOutputs().get(0).getDescriptor());
  Assert.assertEquals(outDesc, v2.getOutputs().get(0).getDescriptor());
  Assert.assertEquals(1, v1.getOutputVertices().size());
  Assert.assertEquals(1, v3.getOutputVertices().size());
  Assert.assertEquals(2, v2.getOutputVertices().size());
  Assert.assertTrue(v1.getOutputVertices().contains(v4));
  Assert.assertTrue(v3.getOutputVertices().contains(v5));
  Assert.assertTrue(v2.getOutputVertices().contains(v4));
  Assert.assertTrue(v2.getOutputVertices().contains(v5));
  Assert.assertEquals(2, v4.getInputVertices().size());
  Assert.assertTrue(v4.getInputVertices().contains(v1));
  Assert.assertTrue(v4.getInputVertices().contains(v2));
  Assert.assertEquals(2, v5.getInputVertices().size());
  Assert.assertTrue(v5.getInputVertices().contains(v2));
  Assert.assertTrue(v5.getInputVertices().contains(v3));
  Assert.assertEquals(1, v4.getGroupInputs().size());
  Assert.assertTrue(v4.getGroupInputs().containsKey(groupName1));
  Assert.assertEquals(1, v5.getGroupInputs().size());
  Assert.assertTrue(v5.getGroupInputs().containsKey(groupName2));
  Assert.assertEquals(2, dag.vertexGroups.size());
}
Example 19
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void testCycle2() {
  IllegalStateException ex=null;
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  Edge e3 = new Edge(v2, v4,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  Edge e4 = new Edge(v3, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.addEdge(e3);
  dag.addEdge(e4);
  try{
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNotNull(ex);
  System.out.println(ex.getMessage());
  Assert.assertTrue(ex.getMessage().startsWith("DAG contains a cycle"));
}
Example 20
Source File: TestDAGVerify.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 5000)
public void testVertexGroupWithMultipleOutputEdges() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  
  DAG dag = new DAG("testDag");
  VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
  OutputDescriptor outDesc = new OutputDescriptor();
  uv12.addOutput("uvOut", outDesc, null);
  
  GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  
  GroupInputEdge e2 = new GroupInputEdge(uv12, v4,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));

  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
  
  Assert.assertEquals(2, v1.getOutputVertices().size());
  Assert.assertEquals(2, v2.getOutputVertices().size());
  Assert.assertTrue(v1.getOutputVertices().contains(v3));
  Assert.assertTrue(v1.getOutputVertices().contains(v4));
  Assert.assertTrue(v2.getOutputVertices().contains(v3));
  Assert.assertTrue(v2.getOutputVertices().contains(v4));
}