org.apache.flink.runtime.minicluster.TestingMiniCluster Java Examples

/**
 * Tests that tasks with a co-location constraint are scheduled in the same
 * slots. In fact it also tests that consumers are scheduled wrt their input
 * location if the co-location constraint is deactivated.
 */
@Test
public void testCoLocationConstraintJobExecution() throws Exception {
	final int numSlotsPerTaskExecutor = 1;
	final int numTaskExecutors = 3;
	final int parallelism = numTaskExecutors * numSlotsPerTaskExecutor;
	final JobGraph jobGraph = createJobGraph(parallelism);

	final TestingMiniClusterConfiguration miniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(numSlotsPerTaskExecutor)
		.setNumTaskManagers(numTaskExecutors)
		.setLocalCommunication(true)
		.build();

	try (TestingMiniCluster miniCluster = new TestingMiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		miniCluster.submitJob(jobGraph).get();

		final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());

		assertThat(jobResultFuture.get().isSuccess(), is(true));
	}
}
 

/**
 * Tests that tasks with a co-location constraint are scheduled in the same
 * slots. In fact it also tests that consumers are scheduled wrt their input
 * location if the co-location constraint is deactivated.
 */
@Test
public void testCoLocationConstraintJobExecution() throws Exception {
	final int numSlotsPerTaskExecutor = 1;
	final int numTaskExecutors = 3;
	final int parallelism = numTaskExecutors * numSlotsPerTaskExecutor;
	final JobGraph jobGraph = createJobGraph(parallelism);

	final TestingMiniClusterConfiguration miniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(numSlotsPerTaskExecutor)
		.setNumTaskManagers(numTaskExecutors)
		.setLocalCommunication(true)
		.build();

	try (TestingMiniCluster miniCluster = new TestingMiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		miniCluster.submitJob(jobGraph).get();

		final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());

		assertThat(jobResultFuture.get().isSuccess(), is(true));
	}
}
 

@SuppressWarnings("OverlyBroadThrowsClause")
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, PIPELINED_REGION_RESTART_STRATEGY_NAME);

	miniCluster = new TestingMiniCluster(
		new Builder()
			.setNumTaskManagers(1)
			.setNumSlotsPerTaskManager(1)
			.setConfiguration(configuration)
			.setRpcServiceSharing(RpcServiceSharing.DEDICATED)
			.build(),
		null);
	miniCluster.start();

	client = new MiniClusterClient(miniCluster);

	lastTaskManagerIndexInMiniCluster = new AtomicInteger(0);

	failureTracker = new GlobalMapFailureTracker(MAP_NUMBER);
}
 

/**
 * Tests that tasks with a co-location constraint are scheduled in the same
 * slots. In fact it also tests that consumers are scheduled wrt their input
 * location if the co-location constraint is deactivated.
 */
@Test
public void testCoLocationConstraintJobExecution() throws Exception {
	final int numSlotsPerTaskExecutor = 1;
	final int numTaskExecutors = 3;
	final int parallelism = numTaskExecutors * numSlotsPerTaskExecutor;
	final JobGraph jobGraph = createJobGraph(parallelism);

	final TestingMiniClusterConfiguration miniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(numSlotsPerTaskExecutor)
		.setNumTaskManagers(numTaskExecutors)
		.setLocalCommunication(true)
		.build();

	try (TestingMiniCluster miniCluster = new TestingMiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		miniCluster.submitJob(jobGraph).get();

		final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());

		assertThat(jobResultFuture.get().isSuccess(), is(true));
	}
}
 

@SuppressWarnings("OverlyBroadThrowsClause")
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, PIPELINED_REGION_RESTART_STRATEGY_NAME);

	miniCluster = new TestingMiniCluster(
		new Builder()
			.setNumTaskManagers(1)
			.setNumSlotsPerTaskManager(1)
			.setConfiguration(configuration)
			.setRpcServiceSharing(RpcServiceSharing.DEDICATED)
			.build(),
		null);
	miniCluster.start();

	client = new MiniClusterClient(miniCluster);

	lastTaskManagerIndexInMiniCluster = new AtomicInteger(0);

	failureTracker = new GlobalMapFailureTracker(MAP_NUMBER);
}
 

@Before
public void setUp() throws Exception {
	networkBufferPool = new NetworkBufferPool(100, 8192);
	testBufferPool = networkBufferPool.createBufferPool(1, Integer.MAX_VALUE);

	final Configuration configuration = new Configuration();
	configuration.setInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES, BACKPRESSURE_NUM_SAMPLES);

	testingMiniCluster = new TestingMiniCluster(new TestingMiniClusterConfiguration.Builder()
		.setNumTaskManagers(JOB_PARALLELISM)
		.setConfiguration(configuration)
		.build());
	testingMiniCluster.start();

	dispatcherGateway = testingMiniCluster.getDispatcherGatewayFuture().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
 

@Before
public void setUp() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.addAll(createBackPressureSamplingConfiguration());
	configuration.addAll(createNetworkBufferConfiguration());

	final TestingMiniClusterConfiguration testingMiniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(NUM_TASKS)
		.setConfiguration(configuration)
		.build();

	testingMiniCluster = new TestingMiniCluster(testingMiniClusterConfiguration);
	testingMiniCluster.start();
	dispatcherGateway = testingMiniCluster.getDispatcherGatewayFuture().get();
}
 

@BeforeClass
public static void setupClass() throws Exception  {
	highAvailabilityServices = new TestingEmbeddedHaServices(TestingUtils.defaultExecutor());

	miniCluster = new TestingMiniCluster(
		new TestingMiniClusterConfiguration.Builder()
			.setNumTaskManagers(NUM_TMS)
			.setNumSlotsPerTaskManager(SLOTS_PER_TM)
			.build(),
		() -> highAvailabilityServices);

	miniCluster.start();
}
 

@Before
public void setup() throws Exception  {
	miniCluster = new TestingMiniCluster(
		new TestingMiniClusterConfiguration.Builder()
			.setNumTaskManagers(NUM_TMS)
			.setNumSlotsPerTaskManager(SLOTS_PER_TM)
			.build(),
		null);

	miniCluster.start();
}
 

@Before
public void setUp() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.addAll(createBackPressureSamplingConfiguration());
	configuration.addAll(createNetworkBufferConfiguration());

	final TestingMiniClusterConfiguration testingMiniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(NUM_TASKS)
		.setConfiguration(configuration)
		.build();

	testingMiniCluster = new TestingMiniCluster(testingMiniClusterConfiguration);
	testingMiniCluster.start();
	dispatcherGateway = testingMiniCluster.getDispatcherGatewayFuture().get();
}
 

@Before
public void setup() throws Exception  {
	miniCluster = new TestingMiniCluster(
		new TestingMiniClusterConfiguration.Builder()
			.setNumTaskManagers(NUM_TMS)
			.setNumSlotsPerTaskManager(SLOTS_PER_TM)
			.build(),
		null);

	miniCluster.start();
}
 

@BeforeClass
public static void setupClass() throws Exception  {
	highAvailabilityServices = new TestingEmbeddedHaServices(TestingUtils.defaultExecutor());

	miniCluster = new TestingMiniCluster(
		new TestingMiniClusterConfiguration.Builder()
			.setNumTaskManagers(NUM_TMS)
			.setNumSlotsPerTaskManager(SLOTS_PER_TM)
			.build(),
		() -> highAvailabilityServices);

	miniCluster.start();
}
 

@BeforeClass
public static void setupClass() throws Exception  {
	highAvailabilityServices = new TestingEmbeddedHaServices(TestingUtils.defaultExecutor());

	miniCluster = new TestingMiniCluster(
		new TestingMiniClusterConfiguration.Builder()
			.setNumTaskManagers(NUM_TMS)
			.setNumSlotsPerTaskManager(SLOTS_PER_TM)
			.build(),
		() -> highAvailabilityServices);

	miniCluster.start();
}
 

@Before
public void setup() throws Exception  {
	miniCluster = new TestingMiniCluster(
		new TestingMiniClusterConfiguration.Builder()
			.setNumTaskManagers(NUM_TMS)
			.setNumSlotsPerTaskManager(SLOTS_PER_TM)
			.build(),
		null);

	miniCluster.start();
}
 

private MiniClusterClient(TestingMiniCluster miniCluster) throws ConfigurationException {
	restAddress = miniCluster.getRestAddress().join();
	executorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClient-IO"));
	restClient = createRestClient();
}
 

private DispatcherGateway getNextLeadingDispatcherGateway(TestingMiniCluster miniCluster, @Nullable String previousLeaderAddress, Deadline timeout) throws Exception {
	CommonTestUtils.waitUntilCondition(() -> !miniCluster.getDispatcherGatewayFuture().get().getAddress().equals(previousLeaderAddress), timeout, 20L);
	return miniCluster.getDispatcherGatewayFuture().get();
}
 

private MiniClusterClient(TestingMiniCluster miniCluster) throws ConfigurationException {
	restAddress = miniCluster.getRestAddress().join();
	executorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClient-IO"));
	restClient = createRestClient();
}