Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionGraph#getFailoverStrategy()
The following examples show how to use
org.apache.flink.runtime.executiongraph.ExecutionGraph#getFailoverStrategy() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that validates that a graph with single unconnected vertices works correctly. * * <pre> * (v1) * * (v2) * * (v3) * * ... * </pre> */ @Test public void testIndividualVertices() throws Exception { final JobVertex source1 = new JobVertex("source1"); source1.setInvokableClass(NoOpInvokable.class); source1.setParallelism(2); final JobVertex source2 = new JobVertex("source2"); source2.setInvokableClass(NoOpInvokable.class); source2.setParallelism(2); final JobGraph jobGraph = new JobGraph("test job", source1, source2); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion sourceRegion11 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[0]); FailoverRegion sourceRegion12 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[1]); FailoverRegion targetRegion21 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[0]); FailoverRegion targetRegion22 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[1]); assertTrue(sourceRegion11 != sourceRegion12); assertTrue(sourceRegion12 != targetRegion21); assertTrue(targetRegion21 != targetRegion22); }
Example 2
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests that validates that a graph with single unconnected vertices works correctly. * * <pre> * (v1) * * (v2) * * (v3) * * ... * </pre> */ @Test public void testIndividualVertices() throws Exception { final JobVertex source1 = new JobVertex("source1"); source1.setInvokableClass(NoOpInvokable.class); source1.setParallelism(2); final JobVertex source2 = new JobVertex("source2"); source2.setInvokableClass(NoOpInvokable.class); source2.setParallelism(2); final JobGraph jobGraph = new JobGraph("test job", source1, source2); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion sourceRegion11 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[0]); FailoverRegion sourceRegion12 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[1]); FailoverRegion targetRegion21 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[0]); FailoverRegion targetRegion22 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[1]); assertTrue(sourceRegion11 != sourceRegion12); assertTrue(sourceRegion12 != targetRegion21); assertTrue(targetRegion21 != targetRegion22); }
Example 3
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * This test checks that are strictly co-located vertices are in the same failover region, * even through they are connected via a blocking pattern. * This is currently an assumption / limitation of the scheduler. */ @Test public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception { final JobVertex source = new JobVertex("source"); source.setInvokableClass(NoOpInvokable.class); source.setParallelism(10); final JobVertex target = new JobVertex("target"); target.setInvokableClass(NoOpInvokable.class); target.setParallelism(10); target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); final SlotSharingGroup sharingGroup = new SlotSharingGroup(); source.setSlotSharingGroup(sharingGroup); target.setSlotSharingGroup(sharingGroup); source.setStrictlyCoLocatedWith(target); final JobGraph jobGraph = new JobGraph("test job", source, target); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]); FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]); FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]); FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]); // we use 'assertTrue' here rather than 'assertEquals' because we want to test // for referential equality, to be on the safe side assertTrue(sourceRegion1 == sourceRegion2); assertTrue(sourceRegion2 == targetRegion1); assertTrue(targetRegion1 == targetRegion2); }
Example 4
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testDiamondWithMixedPipelinedAndBlockingExchanges() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); vertex3.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex4.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex4.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator(); FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); while (evs.hasNext()) { FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); assertTrue(preRegion == region); } }
Example 5
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 5 votes |
/** * <pre> * (a1) -+-> (b1) -+-> (c1) * X X * (a2) -+-> (b2) -+-> (c2) * X X * (a3) -+-> (b3) -+-> (c3) * * ^ ^ * | | * (pipelined) (blocking) * </pre> */ @Test public void testTwoComponentsViaBlockingExchange2() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(3); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(2); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(2); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]); assertTrue(region1 == region2); assertTrue(region2 != region31); assertTrue(region32 != region31); }
Example 6
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 5 votes |
/** * <pre> * (a1) -+-> (b1) -+-> (c1) * X * (a2) -+-> (b2) -+-> (c2) * X * (a3) -+-> (b3) -+-> (c3) * * ^ ^ * | | * (pipelined) (blocking) * * </pre> */ @Test public void testTwoComponentsViaBlockingExchange() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(3); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(2); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(2); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]); assertTrue(region1 == region2); assertTrue(region2 != region31); assertTrue(region32 != region31); }
Example 7
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that validates that a single pipelined component via a sequence of all-to-all * connections works correctly. * * <pre> * (a1) -+-> (b1) -+-> (c1) * X X * (a2) -+-> (b2) -+-> (c2) * X X * (a3) -+-> (b3) -+-> (c3) * * ... * </pre> */ @Test public void testOneComponentViaTwoExchanges() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(3); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(5); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(2); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[4]); FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); assertTrue(region1 == region2); assertTrue(region2 == region3); }
Example 8
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 5 votes |
/** * This test checks that are strictly co-located vertices are in the same failover region, * even through they are connected via a blocking pattern. * This is currently an assumption / limitation of the scheduler. */ @Test public void testBlockingAllToAllTopologyWithCoLocation() throws Exception { final JobVertex source = new JobVertex("source"); source.setInvokableClass(NoOpInvokable.class); source.setParallelism(10); final JobVertex target = new JobVertex("target"); target.setInvokableClass(NoOpInvokable.class); target.setParallelism(13); target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); final SlotSharingGroup sharingGroup = new SlotSharingGroup(); source.setSlotSharingGroup(sharingGroup); target.setSlotSharingGroup(sharingGroup); source.setStrictlyCoLocatedWith(target); final JobGraph jobGraph = new JobGraph("test job", source, target); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]); // we use 'assertTrue' here rather than 'assertEquals' because we want to test // for referential equality, to be on the safe side assertTrue(region1 == region2); }
Example 9
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * This test checks that are strictly co-located vertices are in the same failover region, * even through they are connected via a blocking pattern. * This is currently an assumption / limitation of the scheduler. */ @Test public void testBlockingAllToAllTopologyWithCoLocation() throws Exception { final JobVertex source = new JobVertex("source"); source.setInvokableClass(NoOpInvokable.class); source.setParallelism(10); final JobVertex target = new JobVertex("target"); target.setInvokableClass(NoOpInvokable.class); target.setParallelism(13); target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); final SlotSharingGroup sharingGroup = new SlotSharingGroup(); source.setSlotSharingGroup(sharingGroup); target.setSlotSharingGroup(sharingGroup); source.setStrictlyCoLocatedWith(target); final JobGraph jobGraph = new JobGraph("test job", source, target); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]); // we use 'assertTrue' here rather than 'assertEquals' because we want to test // for referential equality, to be on the safe side assertTrue(region1 == region2); }
Example 10
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testDiamondWithMixedPipelinedAndBlockingExchanges() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); vertex3.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex4.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex4.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator(); FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); while (evs.hasNext()) { FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); assertTrue(preRegion == region); } }
Example 11
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * <pre> * (a1) -+-> (b1) -+-> (c1) * X X * (a2) -+-> (b2) -+-> (c2) * X X * (a3) -+-> (b3) -+-> (c3) * * ^ ^ * | | * (pipelined) (blocking) * </pre> */ @Test public void testTwoComponentsViaBlockingExchange2() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(3); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(2); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(2); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]); assertTrue(region1 == region2); assertTrue(region2 != region31); assertTrue(region32 != region31); }
Example 12
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 5 votes |
/** * This test checks that are strictly co-located vertices are in the same failover region, * even through they are connected via a blocking pattern. * This is currently an assumption / limitation of the scheduler. */ @Test public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception { final JobVertex source = new JobVertex("source"); source.setInvokableClass(NoOpInvokable.class); source.setParallelism(10); final JobVertex target = new JobVertex("target"); target.setInvokableClass(NoOpInvokable.class); target.setParallelism(10); target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); final SlotSharingGroup sharingGroup = new SlotSharingGroup(); source.setSlotSharingGroup(sharingGroup); target.setSlotSharingGroup(sharingGroup); source.setStrictlyCoLocatedWith(target); final JobGraph jobGraph = new JobGraph("test job", source, target); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]); FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]); FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]); FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]); // we use 'assertTrue' here rather than 'assertEquals' because we want to test // for referential equality, to be on the safe side assertTrue(sourceRegion1 == sourceRegion2); assertTrue(sourceRegion2 == targetRegion1); assertTrue(targetRegion1 == targetRegion2); }
Example 13
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests that validates that a single pipelined component via a sequence of all-to-all * connections works correctly. * * <pre> * (a1) -+-> (b1) -+-> (c1) * X X * (a2) -+-> (b2) -+-> (c2) * X X * (a3) -+-> (b3) -+-> (c3) * * ... * </pre> */ @Test public void testOneComponentViaTwoExchanges() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(3); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(5); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(2); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[4]); FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); assertTrue(region1 == region2); assertTrue(region2 == region3); }
Example 14
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Cascades of joins with partially blocking, partially pipelined exchanges: * <pre> * (1)--+ * +--(5)-+ * (2)--+ | * (block) * | * +--(7) * | * (block) * (3)--+ | * +--(6)-+ * (4)--+ * ... * </pre> * * Component 1: 1, 2, 5; component 2: 3,4,6; component 3: 7 */ @Test public void testMultipleComponentsViaCascadeOfJoins() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); final JobVertex vertex5 = new JobVertex("vertex5"); vertex5.setInvokableClass(NoOpInvokable.class); vertex5.setParallelism(4); final JobVertex vertex6 = new JobVertex("vertex6"); vertex6.setInvokableClass(NoOpInvokable.class); vertex6.setParallelism(4); final JobVertex vertex7 = new JobVertex("vertex7"); vertex7.setInvokableClass(NoOpInvokable.class); vertex7.setParallelism(2); vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[5]); FailoverRegion region5 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex5.getID()).getTaskVertices()[2]); assertTrue(region1 == region2); assertTrue(region1 == region5); FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); FailoverRegion region4 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex4.getID()).getTaskVertices()[5]); FailoverRegion region6 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex6.getID()).getTaskVertices()[2]); assertTrue(region3 == region4); assertTrue(region3 == region6); FailoverRegion region71 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[0]); FailoverRegion region72 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[1]); assertTrue(region71 != region72); assertTrue(region1 != region71); assertTrue(region1 != region72); assertTrue(region3 != region71); assertTrue(region3 != region72); }
Example 15
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests that validates that a single pipelined component instance from one source * works correctly. * * <p>Non-parallelized view: * <pre> * +--(1) * +--(5)-+ * | +--(2) * (7)--+ * | +--(3) * +--(6)-+ * +--(4) * ... * </pre> */ @Test public void testOneComponentInstanceFromOneSource() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); final JobVertex vertex5 = new JobVertex("vertex5"); vertex5.setInvokableClass(NoOpInvokable.class); vertex5.setParallelism(4); final JobVertex vertex6 = new JobVertex("vertex6"); vertex6.setInvokableClass(NoOpInvokable.class); vertex6.setParallelism(4); final JobVertex vertex7 = new JobVertex("vertex7"); vertex7.setInvokableClass(NoOpInvokable.class); vertex7.setParallelism(2); vertex1.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex2.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex4.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex5.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex7, vertex5, vertex6, vertex1, vertex2, vertex3, vertex4); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator(); FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); while (evs.hasNext()) { FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); assertTrue(preRegion == region); } }
Example 16
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 4 votes |
/** * Tests that validates that a single pipelined component via a cascade of joins * works correctly. * * <p>Non-parallelized view: * <pre> * (1)--+ * +--(5)-+ * (2)--+ | * +--(7) * (3)--+ | * +--(6)-+ * (4)--+ * ... * </pre> */ @Test public void testOneComponentViaCascadeOfJoins() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); final JobVertex vertex5 = new JobVertex("vertex5"); vertex5.setInvokableClass(NoOpInvokable.class); vertex5.setParallelism(4); final JobVertex vertex6 = new JobVertex("vertex6"); vertex6.setInvokableClass(NoOpInvokable.class); vertex6.setParallelism(4); final JobVertex vertex7 = new JobVertex("vertex7"); vertex7.setInvokableClass(NoOpInvokable.class); vertex7.setParallelism(2); vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator(); FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); while (evs.hasNext()) { FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); assertTrue(preRegion == region); } }
Example 17
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 4 votes |
/** * Tests that validates that a single pipelined component instance from one source * works correctly. * * <p>Non-parallelized view: * <pre> * +--(1) * +--(5)-+ * | +--(2) * (7)--+ * | +--(3) * +--(6)-+ * +--(4) * ... * </pre> */ @Test public void testOneComponentInstanceFromOneSource() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); final JobVertex vertex5 = new JobVertex("vertex5"); vertex5.setInvokableClass(NoOpInvokable.class); vertex5.setParallelism(4); final JobVertex vertex6 = new JobVertex("vertex6"); vertex6.setInvokableClass(NoOpInvokable.class); vertex6.setParallelism(4); final JobVertex vertex7 = new JobVertex("vertex7"); vertex7.setInvokableClass(NoOpInvokable.class); vertex7.setParallelism(2); vertex1.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex2.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex4.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex5.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex7, vertex5, vertex6, vertex1, vertex2, vertex3, vertex4); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator(); FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); while (evs.hasNext()) { FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); assertTrue(preRegion == region); } }
Example 18
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests that validates that a single pipelined component via a cascade of joins * works correctly. * * <p>Non-parallelized view: * <pre> * (1)--+ * +--(5)-+ * (2)--+ | * +--(7) * (3)--+ | * +--(6)-+ * (4)--+ * ... * </pre> */ @Test public void testOneComponentViaCascadeOfJoins() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); final JobVertex vertex5 = new JobVertex("vertex5"); vertex5.setInvokableClass(NoOpInvokable.class); vertex5.setParallelism(4); final JobVertex vertex6 = new JobVertex("vertex6"); vertex6.setInvokableClass(NoOpInvokable.class); vertex6.setParallelism(4); final JobVertex vertex7 = new JobVertex("vertex7"); vertex7.setInvokableClass(NoOpInvokable.class); vertex7.setParallelism(2); vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator(); FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); while (evs.hasNext()) { FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); assertTrue(preRegion == region); } }
Example 19
Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0 | 4 votes |
/** * Cascades of joins with partially blocking, partially pipelined exchanges: * <pre> * (1)--+ * +--(5)-+ * (2)--+ | * (block) * | * +--(7) * | * (block) * (3)--+ | * +--(6)-+ * (4)--+ * ... * </pre> * * Component 1: 1, 2, 5; component 2: 3,4,6; component 3: 7 */ @Test public void testMultipleComponentsViaCascadeOfJoins() throws Exception { final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(8); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(8); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(8); final JobVertex vertex4 = new JobVertex("vertex4"); vertex4.setInvokableClass(NoOpInvokable.class); vertex4.setParallelism(8); final JobVertex vertex5 = new JobVertex("vertex5"); vertex5.setInvokableClass(NoOpInvokable.class); vertex5.setParallelism(4); final JobVertex vertex6 = new JobVertex("vertex6"); vertex6.setInvokableClass(NoOpInvokable.class); vertex6.setParallelism(4); final JobVertex vertex7 = new JobVertex("vertex7"); vertex7.setInvokableClass(NoOpInvokable.class); vertex7.setParallelism(2); vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[5]); FailoverRegion region5 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex5.getID()).getTaskVertices()[2]); assertTrue(region1 == region2); assertTrue(region1 == region5); FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); FailoverRegion region4 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex4.getID()).getTaskVertices()[5]); FailoverRegion region6 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex6.getID()).getTaskVertices()[2]); assertTrue(region3 == region4); assertTrue(region3 == region6); FailoverRegion region71 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[0]); FailoverRegion region72 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[1]); assertTrue(region71 != region72); assertTrue(region1 != region71); assertTrue(region1 != region72); assertTrue(region3 != region71); assertTrue(region3 != region72); }
Example 20
Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests that validates that embarrassingly parallel chains of vertices work correctly. * * <pre> * (a1) --> (b1) * * (a2) --> (b2) * * (a3) --> (b3) * * ... * </pre> */ @Test public void testEmbarrassinglyParallelCase() throws Exception { int parallelism = 10000; final JobVertex vertex1 = new JobVertex("vertex1"); vertex1.setInvokableClass(NoOpInvokable.class); vertex1.setParallelism(parallelism); final JobVertex vertex2 = new JobVertex("vertex2"); vertex2.setInvokableClass(NoOpInvokable.class); vertex2.setParallelism(parallelism); final JobVertex vertex3 = new JobVertex("vertex3"); vertex3.setInvokableClass(NoOpInvokable.class); vertex3.setParallelism(parallelism); vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); final ExecutionGraph eg = createExecutionGraph(jobGraph); RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); FailoverRegion preRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]); FailoverRegion preRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); FailoverRegion preRegion3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); assertTrue(preRegion1 == preRegion2); assertTrue(preRegion2 == preRegion3); for (int i = 1; i < parallelism; ++i) { FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[i]); FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[i]); FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[i]); assertTrue(region1 == region2); assertTrue(region2 == region3); assertTrue(preRegion1 != region1); } }