Java Code Examples for org.apache.flink.runtime.blob.VoidBlobWriter#getInstance()
The following examples show how to use
org.apache.flink.runtime.blob.VoidBlobWriter#getInstance() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ExecutionGraph.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@VisibleForTesting
ExecutionGraph(
JobInformation jobInformation,
ScheduledExecutorService futureExecutor,
Executor ioExecutor,
Time timeout,
RestartStrategy restartStrategy,
FailoverStrategy.Factory failoverStrategy,
SlotProvider slotProvider) throws IOException {
this(
jobInformation,
futureExecutor,
ioExecutor,
timeout,
restartStrategy,
failoverStrategy,
slotProvider,
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
timeout);
}
Example 2
| Source File: LegacySchedulerBatchSchedulingTest.java From flink with Apache License 2.0 | 6 votes |
|
private LegacyScheduler createLegacyScheduler(JobGraph jobGraph, SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor, Time slotRequestTimeout) throws Exception {
final Scheduler scheduler = createScheduler(slotPool, mainThreadExecutor);
final LegacyScheduler legacyScheduler = new LegacyScheduler(
LOG,
jobGraph,
VoidBackPressureStatsTracker.INSTANCE,
TestingUtils.defaultExecutor(),
new Configuration(),
scheduler,
TestingUtils.defaultExecutor(),
getClass().getClassLoader(),
new StandaloneCheckpointRecoveryFactory(),
TestingUtils.TIMEOUT(),
new NoRestartStrategy.NoRestartStrategyFactory(),
VoidBlobWriter.getInstance(),
UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(),
slotRequestTimeout,
NettyShuffleMaster.INSTANCE,
NoOpPartitionTracker.INSTANCE);
legacyScheduler.setMainThreadExecutor(mainThreadExecutor);
return legacyScheduler;
}
Example 3
| Source File: ExecutionGraph.java From flink with Apache License 2.0 | 6 votes |
|
@VisibleForTesting
ExecutionGraph(
JobInformation jobInformation,
ScheduledExecutorService futureExecutor,
Executor ioExecutor,
Time timeout,
RestartStrategy restartStrategy,
FailoverStrategy.Factory failoverStrategy,
SlotProvider slotProvider) throws IOException {
this(
jobInformation,
futureExecutor,
ioExecutor,
timeout,
restartStrategy,
failoverStrategy,
slotProvider,
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
timeout);
}
Example 4
| Source File: TestingJobManagerSharedServicesBuilder.java From flink with Apache License 2.0 | 5 votes |
|
public TestingJobManagerSharedServicesBuilder() {
scheduledExecutorService = TestingUtils.defaultExecutor();
libraryCacheManager = ContextClassLoaderLibraryCacheManager.INSTANCE;
backPressureSampleCoordinator = new BackPressureRequestCoordinator(Runnable::run, 10000);
backPressureStatsTracker = VoidBackPressureStatsTracker.INSTANCE;
blobWriter = VoidBlobWriter.getInstance();
}
Example 5
| Source File: ConcurrentFailoverStrategyExecutionGraphTest.java From flink with Apache License 2.0 | 5 votes |
|
private ExecutionGraph createSampleGraph(
JobID jid,
Factory failoverStrategy,
RestartStrategy restartStrategy,
SlotProvider slotProvider,
int parallelism) throws Exception {
final JobInformation jobInformation = new DummyJobInformation(
jid,
"test job");
// build a simple execution graph with on job vertex, parallelism 2
final Time timeout = Time.seconds(10L);
final ExecutionGraph graph = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
timeout,
restartStrategy,
failoverStrategy,
slotProvider,
getClass().getClassLoader(),
VoidBlobWriter.getInstance(),
timeout);
JobVertex jv = new JobVertex("test vertex");
jv.setInvokableClass(NoOpInvokable.class);
jv.setParallelism(parallelism);
JobGraph jg = new JobGraph(jid, "testjob", jv);
graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
return graph;
}
Example 6
| Source File: TestingJobManagerSharedServicesBuilder.java From flink with Apache License 2.0 | 5 votes |
|
public TestingJobManagerSharedServicesBuilder() {
scheduledExecutorService = TestingUtils.defaultExecutor();
libraryCacheManager = ContextClassLoaderLibraryCacheManager.INSTANCE;
restartStrategyFactory = new NoOrFixedIfCheckpointingEnabledRestartStrategyFactory();
stackTraceSampleCoordinator = mock(StackTraceSampleCoordinator.class);
backPressureStatsTracker = VoidBackPressureStatsTracker.INSTANCE;
blobWriter = VoidBlobWriter.getInstance();
}
Example 7
| Source File: ConcurrentFailoverStrategyExecutionGraphTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private ExecutionGraph createSampleGraph(
JobID jid,
Factory failoverStrategy,
RestartStrategy restartStrategy,
SlotProvider slotProvider,
int parallelism) throws Exception {
final JobInformation jobInformation = new DummyJobInformation(
jid,
"test job");
// build a simple execution graph with on job vertex, parallelism 2
final Time timeout = Time.seconds(10L);
final ExecutionGraph graph = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
timeout,
restartStrategy,
failoverStrategy,
slotProvider,
getClass().getClassLoader(),
VoidBlobWriter.getInstance(),
timeout);
JobVertex jv = new JobVertex("test vertex");
jv.setInvokableClass(NoOpInvokable.class);
jv.setParallelism(parallelism);
JobGraph jg = new JobGraph(jid, "testjob", jv);
graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
return graph;
}
Example 8
| Source File: TestingJobManagerSharedServicesBuilder.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public TestingJobManagerSharedServicesBuilder() {
scheduledExecutorService = TestingUtils.defaultExecutor();
libraryCacheManager = ContextClassLoaderLibraryCacheManager.INSTANCE;
restartStrategyFactory = new NoOrFixedIfCheckpointingEnabledRestartStrategyFactory();
stackTraceSampleCoordinator = mock(StackTraceSampleCoordinator.class);
backPressureStatsTracker = VoidBackPressureStatsTracker.INSTANCE;
blobWriter = VoidBlobWriter.getInstance();
}
Example 9
| Source File: ExecutionGraphCheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
CheckpointIDCounter counter,
CompletedCheckpointStore store) throws Exception {
final Time timeout = Time.days(1L);
ExecutionGraph executionGraph = new ExecutionGraph(
new DummyJobInformation(),
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
timeout,
new NoRestartStrategy(),
new RestartAllStrategy.Factory(),
new TestingSlotProvider(slotRequestId -> CompletableFuture.completedFuture(new TestingLogicalSlot())),
ClassLoader.getSystemClassLoader(),
VoidBlobWriter.getInstance(),
timeout);
executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
executionGraph.enableCheckpointing(
100,
100,
100,
1,
CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList(),
counter,
store,
new MemoryStateBackend(),
CheckpointStatsTrackerTest.createTestTracker());
JobVertex jobVertex = new JobVertex("MockVertex");
jobVertex.setInvokableClass(AbstractInvokable.class);
executionGraph.attachJobGraph(Collections.singletonList(jobVertex));
executionGraph.setQueuedSchedulingAllowed(true);
return executionGraph;
}
Example 10
| Source File: RestartPipelinedRegionStrategyTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v1--->v2-->\
* \
* v4 --->|\
* ----->/ \
* v3-->/ v5
* \ /
* ------------->/
* </pre>
*/
@Test
public void testSingleRegionWithMixedInput() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(3);
v2.setParallelism(2);
v3.setParallelism(2);
v4.setParallelism(5);
v5.setParallelism(2);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
// All in one failover region
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
assertEquals(region1, region2);
assertEquals(region2, region4);
assertEquals(region3, region2);
assertEquals(region1, region5);
}
Example 11
| Source File: RestartPipelinedRegionStrategyTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v1-->v2-->|\
* \
* v4
* /
* v3------>/
* </pre>
*/
@Test
public void testMultiRegionNotAllToAll() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(2);
v2.setParallelism(2);
v3.setParallelism(5);
v4.setParallelism(5);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
// All in one failover region
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
FailoverRegion region11 = strategy.getFailoverRegion(ejv1.getTaskVertices()[0]);
FailoverRegion region12 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
FailoverRegion region21 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
FailoverRegion region22 = strategy.getFailoverRegion(ejv2.getTaskVertices()[1]);
FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
//There should be 3 failover regions. v1 v2 in two, v3 and v4 in one
assertEquals(region11, region21);
assertEquals(region12, region22);
assertFalse(region11.equals(region12));
assertFalse(region3.equals(region4));
}
Example 12
| Source File: RestartPipelinedRegionStrategyTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v2 ------->\
* \
* v1---------> v4 --->|\
* \
* v5
* /
* v3--------------->|/
* </pre>
*/
@Test
public void testMultipleFailoverRegions() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(3);
v2.setParallelism(2);
v3.setParallelism(2);
v4.setParallelism(5);
v5.setParallelism(2);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
// All in one failover region
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
FailoverRegion region31 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region32 = strategy.getFailoverRegion(ejv3.getTaskVertices()[1]);
FailoverRegion region51 = strategy.getFailoverRegion(ejv5.getTaskVertices()[0]);
FailoverRegion region52 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
//There should be 5 failover regions. v1 v2 v4 in one, v3 has two, v5 has two
assertEquals(region1, region2);
assertEquals(region2, region4);
assertFalse(region31.equals(region32));
assertFalse(region51.equals(region52));
}
Example 13
| Source File: RestartPipelinedRegionStrategyTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v1--->v2-->\
* \
* v4 --->\
* ----->/ \
* v3-->/ v5
* \ /
* ------------->/
* </pre>
*/
@Test
public void testSimpleFailoverRegion() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(5);
v2.setParallelism(7);
v3.setParallelism(2);
v4.setParallelism(11);
v5.setParallelism(4);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[2]);
FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[3]);
FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[4]);
FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
assertEquals(region1, region2);
assertEquals(region3, region2);
assertEquals(region4, region2);
assertEquals(region5, region2);
}
Example 14
| Source File: RestartPipelinedRegionStrategyTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v1--->v2-->\
* \
* v4 --->\
* ----->/ \
* v3-->/ v5
* \ /
* ------------->/
* </pre>
*/
@Test
public void testSimpleFailoverRegion() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(5);
v2.setParallelism(7);
v3.setParallelism(2);
v4.setParallelism(11);
v5.setParallelism(4);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[2]);
FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[3]);
FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[4]);
FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
assertEquals(region1, region2);
assertEquals(region3, region2);
assertEquals(region4, region2);
assertEquals(region5, region2);
}
Example 15
| Source File: RestartPipelinedRegionStrategyTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v2 ------->\
* \
* v1---------> v4 --->|\
* \
* v5
* /
* v3--------------->|/
* </pre>
*/
@Test
public void testMultipleFailoverRegions() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(3);
v2.setParallelism(2);
v3.setParallelism(2);
v4.setParallelism(5);
v5.setParallelism(2);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
// All in one failover region
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
FailoverRegion region31 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region32 = strategy.getFailoverRegion(ejv3.getTaskVertices()[1]);
FailoverRegion region51 = strategy.getFailoverRegion(ejv5.getTaskVertices()[0]);
FailoverRegion region52 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
//There should be 5 failover regions. v1 v2 v4 in one, v3 has two, v5 has two
assertEquals(region1, region2);
assertEquals(region2, region4);
assertFalse(region31.equals(region32));
assertFalse(region51.equals(region52));
}
Example 16
| Source File: RestartPipelinedRegionStrategyTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v1--->v2-->\
* \
* v4 --->|\
* ----->/ \
* v3-->/ v5
* \ /
* ------------->/
* </pre>
*/
@Test
public void testSingleRegionWithMixedInput() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(3);
v2.setParallelism(2);
v3.setParallelism(2);
v4.setParallelism(5);
v5.setParallelism(2);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
// All in one failover region
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
assertEquals(region1, region2);
assertEquals(region2, region4);
assertEquals(region3, region2);
assertEquals(region1, region5);
}
Example 17
| Source File: RestartPipelinedRegionStrategyTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Creates a JobGraph of the following form:
*
* <pre>
* v1-->v2-->|\
* \
* v4
* /
* v3------>/
* </pre>
*/
@Test
public void testMultiRegionNotAllToAll() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
v1.setParallelism(2);
v2.setParallelism(2);
v3.setParallelism(5);
v4.setParallelism(5);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4));
final JobInformation jobInformation = new DummyJobInformation(
jobId,
jobName);
ExecutionGraph eg = new ExecutionGraph(
jobInformation,
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new RestartPipelinedRegionStrategy.Factory(),
new TestingSlotProvider(ignored -> new CompletableFuture<>()),
ExecutionGraph.class.getClassLoader(),
VoidBlobWriter.getInstance(),
AkkaUtils.getDefaultTimeout());
try {
eg.attachJobGraph(ordered);
}
catch (JobException e) {
e.printStackTrace();
fail("Job failed with exception: " + e.getMessage());
}
// All in one failover region
RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
FailoverRegion region11 = strategy.getFailoverRegion(ejv1.getTaskVertices()[0]);
FailoverRegion region12 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
FailoverRegion region21 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
FailoverRegion region22 = strategy.getFailoverRegion(ejv2.getTaskVertices()[1]);
FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
//There should be 3 failover regions. v1 v2 in two, v3 and v4 in one
assertEquals(region11, region21);
assertEquals(region12, region22);
assertFalse(region11.equals(region12));
assertFalse(region3.equals(region4));
}
