org.apache.flink.runtime.blob.BlobServer Java Examples

The following examples show how to use org.apache.flink.runtime.blob.BlobServer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DispatcherHATest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Nonnull
private HATestingDispatcher createDispatcher(
	HighAvailabilityServices highAvailabilityServices,
	@Nonnull Queue<DispatcherId> fencingTokens,
	JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	final Configuration configuration = new Configuration();

	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	return new HATestingDispatcher(
		rpcService,
		UUID.randomUUID().toString(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		new BlobServer(configuration, new VoidBlobStore()),
		new HeartbeatServices(1000L, 1000L),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		jobManagerRunnerFactory,
		testingFatalErrorHandler,
		fencingTokens);
}
Example #2
Source File: ExecutionGraphDeploymentWithBlobServerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setupBlobServer() throws IOException {
	Configuration config = new Configuration();
	// always offload the serialized job and task information
	config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
	blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore()));
	blobWriter = blobServer;
	blobCache = blobServer;

	seenHashes.clear();

	// verify that we do not upload the same content more than once
	doAnswer(
		invocation -> {
			PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod();

			assertTrue(seenHashes.add(key.getHash()));

			return key;
		}
	).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any());

	blobServer.start();
}
Example #3
Source File: DispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);

	heartbeatServices = new HeartbeatServices(1000L, 10000L);

	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #4
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void setupClass() throws IOException {
	jobGraph = new JobGraph();

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();

	rpcService = new TestingRpcService();
	configuration = new Configuration();

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #5
Source File: MiniCluster.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
Example #6
Source File: DispatcherHATest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Nonnull
private HATestingDispatcher createDispatcher(
	HighAvailabilityServices highAvailabilityServices,
	@Nonnull Queue<DispatcherId> fencingTokens,
	JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	final Configuration configuration = new Configuration();

	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	return new HATestingDispatcher(
		rpcService,
		UUID.randomUUID().toString(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		new BlobServer(configuration, new VoidBlobStore()),
		new HeartbeatServices(1000L, 1000L),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		jobManagerRunnerFactory,
		testingFatalErrorHandler,
		fencingTokens);
}
Example #7
Source File: PartialDispatcherServicesWithJobGraphStore.java    From flink with Apache License 2.0 6 votes vote down vote up 
public PartialDispatcherServicesWithJobGraphStore(
		@Nonnull Configuration configuration,
		@Nonnull HighAvailabilityServices highAvailabilityServices,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull BlobServer blobServer,
		@Nonnull HeartbeatServices heartbeatServices,
		@Nonnull JobManagerMetricGroupFactory jobManagerMetricGroupFactory,
		@Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore,
		@Nonnull FatalErrorHandler fatalErrorHandler,
		@Nonnull HistoryServerArchivist historyServerArchivist,
		@Nullable String metricQueryServiceAddress,
		@Nonnull JobGraphWriter jobGraphWriter) {
	super(
		configuration,
		highAvailabilityServices,
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroupFactory,
		archivedExecutionGraphStore,
		fatalErrorHandler,
		historyServerArchivist,
		metricQueryServiceAddress);
	this.jobGraphWriter = jobGraphWriter;
}
Example #8
Source File: MiniDispatcherTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void setupClass() throws IOException {
	jobGraph = new JobGraph();

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();

	rpcService = new TestingRpcService();
	configuration = new Configuration();

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #9
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			ioExecutor,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
Example #10
Source File: ExecutionGraphDeploymentWithBlobServerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Before
public void setupBlobServer() throws IOException {
	Configuration config = new Configuration();
	// always offload the serialized job and task information
	config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
	blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore()));
	blobWriter = blobServer;
	blobCache = blobServer;

	seenHashes.clear();

	// verify that we do not upload the same content more than once
	doAnswer(
		invocation -> {
			PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod();

			assertTrue(seenHashes.add(key.getHash()));

			return key;
		}
	).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any());

	blobServer.start();
}
Example #11
Source File: PartialDispatcherServices.java    From flink with Apache License 2.0 6 votes vote down vote up 
public PartialDispatcherServices(
		@Nonnull Configuration configuration,
		@Nonnull HighAvailabilityServices highAvailabilityServices,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull BlobServer blobServer,
		@Nonnull HeartbeatServices heartbeatServices,
		@Nonnull JobManagerMetricGroupFactory jobManagerMetricGroupFactory,
		@Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore,
		@Nonnull FatalErrorHandler fatalErrorHandler,
		@Nonnull HistoryServerArchivist historyServerArchivist,
		@Nullable String metricQueryServiceAddress) {
	this.configuration = configuration;
	this.highAvailabilityServices = highAvailabilityServices;
	this.resourceManagerGatewayRetriever = resourceManagerGatewayRetriever;
	this.blobServer = blobServer;
	this.heartbeatServices = heartbeatServices;
	this.jobManagerMetricGroupFactory = jobManagerMetricGroupFactory;
	this.archivedExecutionGraphStore = archivedExecutionGraphStore;
	this.fatalErrorHandler = fatalErrorHandler;
	this.historyServerArchivist = historyServerArchivist;
	this.metricQueryServiceAddress = metricQueryServiceAddress;
}
Example #12
Source File: ExecutionGraphDeploymentWithBlobServerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setupBlobServer() throws IOException {
	Configuration config = new Configuration();
	// always offload the serialized job and task information
	config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
	blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore()));
	blobWriter = blobServer;
	blobCache = blobServer;

	seenHashes.clear();

	// verify that we do not upload the same content more than once
	doAnswer(
		invocation -> {
			PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod();

			assertTrue(seenHashes.add(key.getHash()));

			return key;
		}
	).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any());

	blobServer.start();
}
Example #13
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
Example #14
Source File: DispatcherServices.java    From flink with Apache License 2.0 6 votes vote down vote up 
public DispatcherServices(
		@Nonnull Configuration configuration,
		@Nonnull HighAvailabilityServices highAvailabilityServices,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull BlobServer blobServer,
		@Nonnull HeartbeatServices heartbeatServices,
		@Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore,
		@Nonnull FatalErrorHandler fatalErrorHandler,
		@Nonnull HistoryServerArchivist historyServerArchivist,
		@Nullable String metricQueryServiceAddress,
		@Nonnull JobManagerMetricGroup jobManagerMetricGroup,
		@Nonnull JobGraphWriter jobGraphWriter,
		@Nonnull JobManagerRunnerFactory jobManagerRunnerFactory) {
	this.configuration = configuration;
	this.highAvailabilityServices = highAvailabilityServices;
	this.resourceManagerGatewayRetriever = resourceManagerGatewayRetriever;
	this.blobServer = blobServer;
	this.heartbeatServices = heartbeatServices;
	this.archivedExecutionGraphStore = archivedExecutionGraphStore;
	this.fatalErrorHandler = fatalErrorHandler;
	this.historyServerArchivist = historyServerArchivist;
	this.metricQueryServiceAddress = metricQueryServiceAddress;
	this.jobManagerMetricGroup = jobManagerMetricGroup;
	this.jobGraphWriter = jobGraphWriter;
	this.jobManagerRunnerFactory = jobManagerRunnerFactory;
}
Example #15
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void setupClass() throws IOException {
	jobGraph = new JobGraph();

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();

	rpcService = new TestingRpcService();
	configuration = new Configuration();

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #16
Source File: AbstractTaskManagerFileHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setup() throws IOException, HandlerRequestException {
	final Configuration configuration = new Configuration();
	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());

	handlerRequest = new HandlerRequest<>(
		EmptyRequestBody.getInstance(),
		new TaskManagerFileMessageParameters(),
		Collections.singletonMap(TaskManagerIdPathParameter.KEY, EXPECTED_TASK_MANAGER_ID.getResourceIdString()),
		Collections.emptyMap());
}
Example #17
Source File: ZooKeeperDefaultDispatcherRunnerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws IOException {
	fatalErrorHandler = new TestingFatalErrorHandler();
	configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getAbsolutePath());

	clusterHaStorageDir = new File(HighAvailabilityServicesUtils.getClusterHighAvailableStoragePath(configuration).toString());
	blobServer = new BlobServer(configuration, BlobUtils.createBlobStoreFromConfig(configuration));
}
Example #18
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			commonRpcService = AkkaRpcServiceUtils.createRemoteRpcService(
				configuration,
				configuration.getString(JobManagerOptions.ADDRESS),
				getRPCPortRange(configuration),
				configuration.getString(JobManagerOptions.BIND_HOST),
				configuration.getOptional(JobManagerOptions.RPC_BIND_PORT));

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				ClusterEntrypointUtils.getPoolSize(configuration),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration, pluginManager);

			final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress());
			metricRegistry.startQueryService(metricQueryServiceRpcService, null);

			final String hostname = RpcUtils.getHostname(commonRpcService);

			processMetricGroup = MetricUtils.instantiateProcessMetricGroup(
				metricRegistry,
				hostname,
				ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
		}
	}
Example #19
Source File: DispatcherResourceManagerComponentFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
DispatcherResourceManagerComponent create(
Configuration configuration,
Executor ioExecutor,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
ArchivedExecutionGraphStore archivedExecutionGraphStore,
MetricQueryServiceRetriever metricQueryServiceRetriever,
FatalErrorHandler fatalErrorHandler) throws Exception;
Example #20
Source File: TestingMiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected Collection<? extends DispatcherResourceManagerComponent> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();

	final List<DispatcherResourceManagerComponent> result = new ArrayList<>(numberDispatcherResourceManagerComponents);

	for (int i = 0; i < numberDispatcherResourceManagerComponents; i++) {
		result.add(
			dispatcherResourceManagerComponentFactory.create(
				configuration,
				getIOExecutor(),
				rpcServiceFactory.createRpcService(),
				haServices,
				blobServer,
				heartbeatServices,
				metricRegistry,
				new MemoryArchivedExecutionGraphStore(),
				metricQueryServiceRetriever,
				fatalErrorHandler));
	}

	return result;
}
Example #21
Source File: JobDispatcherFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public MiniDispatcher createDispatcher(
		Configuration configuration,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		JobManagerMetricGroup jobManagerMetricGroup,
		@Nullable String metricQueryServicePath,
		ArchivedExecutionGraphStore archivedExecutionGraphStore,
		FatalErrorHandler fatalErrorHandler,
		HistoryServerArchivist historyServerArchivist) throws Exception {
	final JobGraph jobGraph = jobGraphRetriever.retrieveJobGraph(configuration);

	final String executionModeValue = configuration.getString(EXECUTION_MODE);

	final ClusterEntrypoint.ExecutionMode executionMode = ClusterEntrypoint.ExecutionMode.valueOf(executionModeValue);

	return new MiniDispatcher(
		rpcService,
		getEndpointId(),
		configuration,
		highAvailabilityServices,
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServicePath,
		archivedExecutionGraphStore,
		DefaultJobManagerRunnerFactory.INSTANCE,
		fatalErrorHandler,
		historyServerArchivist,
		jobGraph,
		executionMode);
}
Example #22
Source File: JobSubmitHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws IOException {
	Configuration config = new Configuration(configuration);
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		TEMPORARY_FOLDER.newFolder().getAbsolutePath());

	blobServer = new BlobServer(config, new VoidBlobStore());
	blobServer.start();
}
Example #23
Source File: AbstractTaskManagerFileHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setup() throws IOException, HandlerRequestException {
	final Configuration configuration = new Configuration();
	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());

	handlerRequest = new HandlerRequest<>(
		EmptyRequestBody.getInstance(),
		new TaskManagerMessageParameters(),
		Collections.singletonMap(TaskManagerIdPathParameter.KEY, EXPECTED_TASK_MANAGER_ID.getResourceIdString()),
		Collections.emptyMap());
}
Example #24
Source File: ClientUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setup() throws IOException {
	Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	blobServer = new BlobServer(config, new VoidBlobStore());
	blobServer.start();
}
Example #25
Source File: BlobServerResource.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void before() throws Throwable {
	temporaryFolder.create();

	Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(config, new VoidBlobStore());
	blobServer.start();
}
Example #26
Source File: DispatcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #27
Source File: ZooKeeperHADispatcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setupClass() throws IOException {
	configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, TEMPORARY_FOLDER.newFolder().getAbsolutePath());
	rpcService = new TestingRpcService();
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #28
Source File: TestingDispatcher.java    From flink with Apache License 2.0 5 votes vote down vote up 
TestingDispatcher(
	RpcService rpcService,
	String endpointId,
	Configuration configuration,
	HighAvailabilityServices highAvailabilityServices,
	GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
	BlobServer blobServer,
	HeartbeatServices heartbeatServices,
	JobManagerMetricGroup jobManagerMetricGroup,
	@Nullable String metricQueryServiceAddress,
	ArchivedExecutionGraphStore archivedExecutionGraphStore,
	JobManagerRunnerFactory jobManagerRunnerFactory,
	FatalErrorHandler fatalErrorHandler) throws Exception {
	super(
		rpcService,
		endpointId,
		configuration,
		highAvailabilityServices,
		highAvailabilityServices.getSubmittedJobGraphStore(),
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServiceAddress,
		archivedExecutionGraphStore,
		jobManagerRunnerFactory,
		fatalErrorHandler,
		VoidHistoryServerArchivist.INSTANCE);

	this.startFuture = new CompletableFuture<>();
}
Example #29
Source File: DispatcherHATest.java    From flink with Apache License 2.0 5 votes vote down vote up 
HATestingDispatcher(
		RpcService rpcService,
		String endpointId,
		Configuration configuration,
		HighAvailabilityServices highAvailabilityServices,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		JobManagerMetricGroup jobManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		ArchivedExecutionGraphStore archivedExecutionGraphStore,
		JobManagerRunnerFactory jobManagerRunnerFactory,
		FatalErrorHandler fatalErrorHandler,
		@Nonnull Queue<DispatcherId> fencingTokens) throws Exception {
	super(
		rpcService,
		endpointId,
		configuration,
		highAvailabilityServices,
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServiceAddress,
		archivedExecutionGraphStore,
		jobManagerRunnerFactory,
		fatalErrorHandler);
	this.fencingTokens = fencingTokens;
}
Example #30
Source File: ExecutionGraphDeploymentWithBlobCacheTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
@Override
public void setupBlobServer() throws IOException {
	Configuration config = new Configuration();
	// always offload the serialized job and task information
	config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
	blobServer = new BlobServer(config, new VoidBlobStore());
	blobServer.start();
	blobWriter = blobServer;

	InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
	blobCache = new PermanentBlobCache(config, new VoidBlobStore(), serverAddress);
}