org.apache.flink.test.util.MiniClusterWithClientResource Java Examples
The following examples show how to use
org.apache.flink.test.util.MiniClusterWithClientResource.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RescalingITCase.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { // detect parameter change if (currentBackend != backend) { shutDownExistingCluster(); currentBackend = backend; Configuration config = new Configuration(); final File checkpointDir = temporaryFolder.newFolder(); final File savepointDir = temporaryFolder.newFolder(); config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend); config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlots) .build()); cluster.before(); } }
Example #2
Source File: SavepointITCase.java From flink with Apache License 2.0 | 6 votes |
private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000); final JobID jobId = jobGraph.getJobID(); StatefulCounter.resetForTest(parallelism); MiniClusterWithClientResource cluster = clusterFactory.get(); cluster.before(); ClusterClient<?> client = cluster.getClusterClient(); try { client.setDetached(true); client.submitJob(jobGraph, SavepointITCase.class.getClassLoader()); StatefulCounter.getProgressLatch().await(); return client.cancelWithSavepoint(jobId, null); } finally { cluster.after(); StatefulCounter.resetForTest(parallelism); } }
Example #3
Source File: RescalingITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { // detect parameter change if (currentBackend != backend) { shutDownExistingCluster(); currentBackend = backend; Configuration config = new Configuration(); final File checkpointDir = temporaryFolder.newFolder(); final File savepointDir = temporaryFolder.newFolder(); config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend); config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlots) .build()); cluster.before(); } }
Example #4
Source File: RegionFailoverITCase.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { Configuration configuration = new Configuration(); configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "region"); // global failover times: 3, region failover times: NUM_OF_RESTARTS configuration.setInteger(FailingRestartStrategy.NUM_FAILURES_CONFIG_OPTION, 3); configuration.setString(ConfigConstants.RESTART_STRATEGY, FailingRestartStrategy.class.getName()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(configuration) .setNumberTaskManagers(2) .setNumberSlotsPerTaskManager(2).build()); cluster.before(); jobFailedCnt.set(0); numCompletedCheckpoints.set(0); }
Example #5
Source File: NettyEpollITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testNettyEpoll() throws Exception { MiniClusterWithClientResource cluster = trySetUpCluster(); try { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(NUM_TASK_MANAGERS); env.getConfig().disableSysoutLogging(); DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42); input.keyBy(new KeySelector<Integer, Integer>() { @Override public Integer getKey(Integer value) throws Exception { return value; } }) .sum(0) .print(); env.execute(); } finally { cluster.after(); } }
Example #6
Source File: NettyEpollITCase.java From flink with Apache License 2.0 | 6 votes |
private MiniClusterWithClientResource trySetUpCluster() throws Exception { try { Configuration config = new Configuration(); config.setString(NettyShuffleEnvironmentOptions.TRANSPORT_TYPE, "epoll"); MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(NUM_TASK_MANAGERS) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); return cluster; } catch (UnsatisfiedLinkError ex) { // If we failed to init netty because we are not on Linux platform, abort the test. if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) { throw new AssumptionViolatedException("This test is only supported on linux"); } throw ex; } }
Example #7
Source File: HistoryServerTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { jmDirectory = TMP.newFolder("jm_" + versionLessThan14); hsDirectory = TMP.newFolder("hs_" + versionLessThan14); Configuration clusterConfig = new Configuration(); clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(clusterConfig) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); }
Example #8
Source File: HAQueryableStateRocksDBBackendITCase.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
Example #9
Source File: HAQueryableStateFsBackendITCase.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
Example #10
Source File: StreamFaultToleranceTestBase.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { Configuration configuration = new Configuration(); switch (failoverStrategy) { case RestartPipelinedRegionStrategy: configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "region"); break; case RestartAllStrategy: configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "full"); } cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(configuration) .setNumberTaskManagers(NUM_TASK_MANAGERS) .setNumberSlotsPerTaskManager(NUM_TASK_SLOTS) .build()); cluster.before(); }
Example #11
Source File: NotifyCheckpointAbortedITCase.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { Configuration configuration = new Configuration(); configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true); configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName()); checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(configuration) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1).build()); cluster.before(); NormalMap.reset(); DeclineSink.reset(); TestingCompletedCheckpointStore.reset(); }
Example #12
Source File: SavepointITCase.java From flink with Apache License 2.0 | 6 votes |
private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000); final JobID jobId = jobGraph.getJobID(); StatefulCounter.resetForTest(parallelism); MiniClusterWithClientResource cluster = clusterFactory.get(); cluster.before(); ClusterClient<?> client = cluster.getClusterClient(); try { ClientUtils.submitJob(client, jobGraph); StatefulCounter.getProgressLatch().await(); return client.cancelWithSavepoint(jobId, null).get(); } finally { cluster.after(); StatefulCounter.resetForTest(parallelism); } }
Example #13
Source File: StreamFaultToleranceTestBase.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { Configuration configuration = new Configuration(); switch (failoverStrategy) { case RestartPipelinedRegionFailoverStrategy: configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "region"); break; case RestartAllFailoverStrategy: configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "full"); } cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(configuration) .setNumberTaskManagers(NUM_TASK_MANAGERS) .setNumberSlotsPerTaskManager(NUM_TASK_SLOTS) .build()); cluster.before(); }
Example #14
Source File: NettyEpollITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testNettyEpoll() throws Exception { MiniClusterWithClientResource cluster = trySetUpCluster(); try { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(NUM_TASK_MANAGERS); DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42); input.keyBy(new KeySelector<Integer, Integer>() { @Override public Integer getKey(Integer value) throws Exception { return value; } }) .sum(0) .print(); env.execute(); } finally { cluster.after(); } }
Example #15
Source File: NettyEpollITCase.java From flink with Apache License 2.0 | 6 votes |
private MiniClusterWithClientResource trySetUpCluster() throws Exception { try { Configuration config = new Configuration(); config.setString(NettyShuffleEnvironmentOptions.TRANSPORT_TYPE, "epoll"); MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(NUM_TASK_MANAGERS) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); return cluster; } catch (UnsatisfiedLinkError ex) { // If we failed to init netty because we are not on Linux platform, abort the test. if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) { throw new AssumptionViolatedException("This test is only supported on linux"); } throw ex; } }
Example #16
Source File: HistoryServerTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { jmDirectory = tmpFolder.newFolder("jm_" + versionLessThan14); hsDirectory = tmpFolder.newFolder("hs_" + versionLessThan14); Configuration clusterConfig = new Configuration(); clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(clusterConfig) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); }
Example #17
Source File: HAQueryableStateRocksDBBackendITCase.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
Example #18
Source File: HAQueryableStateFsBackendITCase.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
Example #19
Source File: RescalingITCase.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { // detect parameter change if (currentBackend != backend) { shutDownExistingCluster(); currentBackend = backend; Configuration config = new Configuration(); final File checkpointDir = temporaryFolder.newFolder(); final File savepointDir = temporaryFolder.newFolder(); config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend); config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlots) .build()); cluster.before(); } }
Example #20
Source File: HAQueryableStateFsBackendITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
Example #21
Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000); final JobID jobId = jobGraph.getJobID(); StatefulCounter.resetForTest(parallelism); MiniClusterWithClientResource cluster = clusterFactory.get(); cluster.before(); ClusterClient<?> client = cluster.getClusterClient(); try { client.setDetached(true); client.submitJob(jobGraph, SavepointITCase.class.getClassLoader()); StatefulCounter.getProgressLatch().await(); return client.cancelWithSavepoint(jobId, null); } finally { cluster.after(); StatefulCounter.resetForTest(parallelism); } }
Example #22
Source File: NettyEpollITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testNettyEpoll() throws Exception { MiniClusterWithClientResource cluster = trySetUpCluster(); try { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(NUM_TASK_MANAGERS); env.getConfig().disableSysoutLogging(); DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42); input.keyBy(new KeySelector<Integer, Integer>() { @Override public Integer getKey(Integer value) throws Exception { return value; } }) .sum(0) .print(); env.execute(); } finally { cluster.after(); } }
Example #23
Source File: NettyEpollITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private MiniClusterWithClientResource trySetUpCluster() throws Exception { try { Configuration config = new Configuration(); config.setString(TRANSPORT_TYPE, "epoll"); MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(NUM_TASK_MANAGERS) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); return cluster; } catch (UnsatisfiedLinkError ex) { // If we failed to init netty because we are not on Linux platform, abort the test. if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) { throw new AssumptionViolatedException("This test is only supported on linux"); } throw ex; } }
Example #24
Source File: HAQueryableStateRocksDBBackendITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
Example #25
Source File: HistoryServerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { jmDirectory = TMP.newFolder("jm_" + versionLessThan14); hsDirectory = TMP.newFolder("hs_" + versionLessThan14); Configuration clusterConfig = new Configuration(); clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(clusterConfig) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); }
Example #26
Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 5 votes |
protected final MiniClusterWithClientResource getMiniClusterResource() { return new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfigurationSafe()) .setNumberTaskManagers(NUM_OF_TASK_MANAGERS) .setNumberSlotsPerTaskManager(PARALLELISM / NUM_OF_TASK_MANAGERS) .build()); }
Example #27
Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
MiniClusterWithClientResource get() { return new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlotsPerTaskManager) .build()); }
Example #28
Source File: SavepointMigrationTestBase.java From flink with Apache License 2.0 | 5 votes |
protected SavepointMigrationTestBase() throws Exception { miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfiguration()) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(DEFAULT_PARALLELISM) .build()); }
Example #29
Source File: SavepointITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTriggerSavepointForNonExistingJob() throws Exception { // Config final int numTaskManagers = 1; final int numSlotsPerTaskManager = 1; final Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlotsPerTaskManager) .build()); cluster.before(); final ClusterClient<?> client = cluster.getClusterClient(); final JobID jobID = new JobID(); try { client.triggerSavepoint(jobID, null).get(); fail(); } catch (ExecutionException e) { assertTrue(ExceptionUtils.findThrowable(e, FlinkJobNotFoundException.class).isPresent()); assertTrue(ExceptionUtils.findThrowableWithMessage(e, jobID.toString()).isPresent()); } finally { cluster.after(); } }
Example #30
Source File: SavepointITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTriggerSavepointWithCheckpointingDisabled() throws Exception { // Config final int numTaskManagers = 1; final int numSlotsPerTaskManager = 1; final Configuration config = new Configuration(); final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlotsPerTaskManager) .build()); cluster.before(); final ClusterClient<?> client = cluster.getClusterClient(); final JobVertex vertex = new JobVertex("Blocking vertex"); vertex.setInvokableClass(BlockingNoOpInvokable.class); vertex.setParallelism(1); final JobGraph graph = new JobGraph(vertex); try { ClientUtils.submitJob(client, graph); client.triggerSavepoint(graph.getJobID(), null).get(); fail(); } catch (ExecutionException e) { assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent()); assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent()); assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent()); } finally { cluster.after(); } }