Java Code Examples for org.apache.flink.runtime.minicluster.MiniCluster#start()

The following examples show how to use org.apache.flink.runtime.minicluster.MiniCluster#start() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LocalExecutor.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
Example 2
Source File: LocalExecutor.java    From flink with Apache License 2.0 6 votes vote down vote up 
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
Example 3
Source File: FlinkSavepointTest.java    From beam with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  Configuration config = new Configuration();
  // Avoid port collision in parallel tests
  config.setInteger(RestOptions.PORT, 0);
  config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");

  String savepointPath = "file://" + tempFolder.getRoot().getAbsolutePath();
  LOG.info("Savepoints will be written to {}", savepointPath);
  // It is necessary to configure the checkpoint directory for the state backend,
  // even though we only create savepoints in this test.
  config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, savepointPath);
  // Checkpoints will go into a subdirectory of this directory
  config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointPath);

  MiniClusterConfiguration clusterConfig =
      new MiniClusterConfiguration.Builder()
          .setConfiguration(config)
          .setNumTaskManagers(2)
          .setNumSlotsPerTaskManager(2)
          .build();

  flinkCluster = new MiniCluster(clusterConfig);
  flinkCluster.start();
}
Example 4
Source File: PerJobMiniClusterFactory.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Starts a {@link MiniCluster} and submits a job.
 */
public CompletableFuture<JobClient> submitJob(JobGraph jobGraph) throws Exception {
	MiniClusterConfiguration miniClusterConfig = getMiniClusterConfig(jobGraph.getMaximumParallelism());
	MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
	miniCluster.start();

	return miniCluster
		.submitJob(jobGraph)
		.thenApply(result -> new PerJobMiniClusterJobClient(result.getJobID(), miniCluster))
		.whenComplete((ignored, throwable) -> {
			if (throwable != null) {
				// We failed to create the JobClient and must shutdown to ensure cleanup.
				shutDownCluster(miniCluster);
			}
		})
		.thenApply(Function.identity());
}
Example 5
Source File: JobManagerWatermarkTrackerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setUp() throws Exception {
	final Configuration config = new Configuration();
	config.setInteger(RestOptions.PORT, 0);

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(config)
		.setNumTaskManagers(1)
		.setNumSlotsPerTaskManager(1)
		.build();

	flink = new MiniCluster(miniClusterConfiguration);

	flink.start();
}
Example 6
Source File: MiniClusterResource.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
Example 7
Source File: JobManagerWatermarkTrackerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setUp() throws Exception {
	final Configuration config = new Configuration();
	config.setInteger(RestOptions.PORT, 0);

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(config)
		.setNumTaskManagers(1)
		.setNumSlotsPerTaskManager(1)
		.build();

	flink = new MiniCluster(miniClusterConfiguration);

	flink.start();
}
Example 8
Source File: MiniClusterResource.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
Example 9
Source File: RemoteChannelThroughputBenchmark.java    From flink-benchmarks with Apache License 2.0 5 votes vote down vote up 
@Setup
public void setUp() throws Exception {
    MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
        .setNumTaskManagers(NUM_VERTICES * PARALLELISM)
        .setNumSlotsPerTaskManager(1)
        .build();
    miniCluster = new MiniCluster(miniClusterConfiguration);
    miniCluster.start();
}
Example 10
Source File: FlinkRequiresStableInputTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  final int parallelism = 1;

  Configuration config = new Configuration();
  // Avoid port collision in parallel tests
  config.setInteger(RestOptions.PORT, 0);
  config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
  // It is necessary to configure the checkpoint directory for the state backend,
  // even though we only create savepoints in this test.
  config.setString(
      CheckpointingOptions.CHECKPOINTS_DIRECTORY,
      "file://" + tempFolder.getRoot().getAbsolutePath());
  // Checkpoints will go into a subdirectory of this directory
  config.setString(
      CheckpointingOptions.SAVEPOINT_DIRECTORY,
      "file://" + tempFolder.getRoot().getAbsolutePath());

  MiniClusterConfiguration clusterConfig =
      new MiniClusterConfiguration.Builder()
          .setConfiguration(config)
          .setNumTaskManagers(1)
          .setNumSlotsPerTaskManager(1)
          .build();

  flinkCluster = new MiniCluster(clusterConfig);
  flinkCluster.start();

  TestStreamEnvironment.setAsContext(flinkCluster, parallelism);
}
Example 11
Source File: JobManagerWatermarkTrackerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setUp() throws Exception {
	final Configuration config = new Configuration();
	config.setInteger(RestOptions.PORT, 0);

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(config)
		.setNumTaskManagers(1)
		.setNumSlotsPerTaskManager(1)
		.build();

	flink = new MiniCluster(miniClusterConfiguration);

	flink.start();
}
Example 12
Source File: CoordinatorEventsExactlyOnceITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void startMiniCluster() throws Exception {
	final Configuration config = new Configuration();
	config.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration clusterCfg = new MiniClusterConfiguration.Builder()
		.setNumTaskManagers(2)
		.setNumSlotsPerTaskManager(1)
		.setConfiguration(config)
		.build();

	miniCluster = new MiniCluster(clusterCfg);
	miniCluster.start();
}
Example 13
Source File: MiniClusterResource.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
Example 14
Source File: LocalStreamEnvironment.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
 * specified name.
 *
 * @param jobName
 *            name of the job
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	// transform the streaming program into a JobGraph
	StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);

	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}
Example 15
Source File: LocalStreamEnvironment.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Executes the JobGraph of the on a mini cluster of ClusterUtil with a user
 * specified name.
 *
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}