org.apache.flink.runtime.blob.VoidBlobStore Java Examples
The following examples show how to use
org.apache.flink.runtime.blob.VoidBlobStore.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ZooKeeperLeaderRetrievalTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Before public void before() throws Exception { testingServer = new TestingServer(); config = new Configuration(); config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString()); CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config); highAvailabilityServices = new ZooKeeperHaServices( client, TestingUtils.defaultExecutor(), config, new VoidBlobStore()); }
Example #2
Source File: TaskExecutorTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws IOException { rpc = new TestingRpcService(); timerService = new TimerService<>(TestingUtils.defaultExecutor(), timeout.toMilliseconds()); dummyBlobCacheService = new BlobCacheService( new Configuration(), new VoidBlobStore(), null); configuration = new Configuration(); taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration); taskManagerLocation = new LocalTaskManagerLocation(); jobId = new JobID(); testingFatalErrorHandler = new TestingFatalErrorHandler(); haServices = new TestingHighAvailabilityServices(); resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(); jobManagerLeaderRetriever = new SettableLeaderRetrievalService(); haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever); haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever); nettyShuffleEnvironment = new NettyShuffleEnvironmentBuilder().build(); }
Example #3
Source File: MiniDispatcherTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setupClass() throws IOException { jobGraph = new JobGraph(); archivedExecutionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobGraph.getJobID()) .setState(JobStatus.FINISHED) .build(); rpcService = new TestingRpcService(); configuration = new Configuration(); configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(configuration, new VoidBlobStore()); }
Example #4
Source File: ExecutionGraphDeploymentWithBlobServerTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setupBlobServer() throws IOException { Configuration config = new Configuration(); // always offload the serialized job and task information config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0); blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore())); blobWriter = blobServer; blobCache = blobServer; seenHashes.clear(); // verify that we do not upload the same content more than once doAnswer( invocation -> { PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod(); assertTrue(seenHashes.add(key.getHash())); return key; } ).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any()); blobServer.start(); }
Example #5
Source File: DispatcherHATest.java From flink with Apache License 2.0 | 6 votes |
@Nonnull private HATestingDispatcher createDispatcher( HighAvailabilityServices highAvailabilityServices, @Nonnull Queue<DispatcherId> fencingTokens, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception { final Configuration configuration = new Configuration(); TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway(); return new HATestingDispatcher( rpcService, UUID.randomUUID().toString(), configuration, highAvailabilityServices, () -> CompletableFuture.completedFuture(resourceManagerGateway), new BlobServer(configuration, new VoidBlobStore()), new HeartbeatServices(1000L, 1000L), UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), null, new MemoryArchivedExecutionGraphStore(), jobManagerRunnerFactory, testingFatalErrorHandler, fencingTokens); }
Example #6
Source File: ZooKeeperLeaderRetrievalTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void before() throws Exception { testingServer = new TestingServer(); config = new Configuration(); config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString()); CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config); highAvailabilityServices = new ZooKeeperHaServices( client, TestingUtils.defaultExecutor(), config, new VoidBlobStore()); }
Example #7
Source File: ZooKeeperLeaderRetrievalTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void before() throws Exception { testingServer = new TestingServer(); config = new Configuration(); config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString()); CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config); highAvailabilityServices = new ZooKeeperHaServices( client, TestingUtils.defaultExecutor(), config, new VoidBlobStore()); }
Example #8
Source File: DispatcherTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { final JobVertex testVertex = new JobVertex("testVertex"); testVertex.setInvokableClass(NoOpInvokable.class); jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex); heartbeatServices = new HeartbeatServices(1000L, 10000L); jobMasterLeaderElectionService = new TestingLeaderElectionService(); haServices = new TestingHighAvailabilityServices(); haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService); haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory()); haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService()); configuration = new Configuration(); configuration.setString( BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); createdJobManagerRunnerLatch = new CountDownLatch(2); blobServer = new BlobServer(configuration, new VoidBlobStore()); }
Example #9
Source File: MiniDispatcherTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setupClass() throws IOException { jobGraph = new JobGraph(); archivedExecutionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobGraph.getJobID()) .setState(JobStatus.FINISHED) .build(); rpcService = new TestingRpcService(); configuration = new Configuration(); configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(configuration, new VoidBlobStore()); }
Example #10
Source File: TaskExecutorSlotLifetimeTest.java From flink with Apache License 2.0 | 6 votes |
private TaskExecutor createTaskExecutor(Configuration configuration, TestingRpcService rpcService, TestingHighAvailabilityServices haServices, LocalUnresolvedTaskManagerLocation unresolvedTaskManagerLocation) throws IOException { return new TaskExecutor( rpcService, TaskManagerConfiguration.fromConfiguration( configuration, TaskExecutorResourceUtils.resourceSpecFromConfigForLocalExecution(configuration), InetAddress.getLoopbackAddress().getHostAddress()), haServices, new TaskManagerServicesBuilder() .setTaskSlotTable(TaskSlotUtils.createTaskSlotTable(1)) .setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation) .build(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestingHeartbeatServices(), UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), null, new BlobCacheService( configuration, new VoidBlobStore(), null), testingFatalErrorHandlerResource.getFatalErrorHandler(), new TestingTaskExecutorPartitionTracker(), TaskManagerRunner.createBackPressureSampleService(configuration, rpcService.getScheduledExecutor())); }
Example #11
Source File: TaskManagerRunnerStartupTest.java From flink with Apache License 2.0 | 6 votes |
private static void startTaskManager( Configuration configuration, RpcService rpcService, HighAvailabilityServices highAvailabilityServices ) throws Exception { TaskManagerRunner.startTaskManager( configuration, ResourceID.generate(), rpcService, highAvailabilityServices, new TestingHeartbeatServices(), NoOpMetricRegistry.INSTANCE, new BlobCacheService( configuration, new VoidBlobStore(), null), false, ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, error -> {}); }
Example #12
Source File: ExecutionGraphDeploymentWithBlobServerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Before public void setupBlobServer() throws IOException { Configuration config = new Configuration(); // always offload the serialized job and task information config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0); blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore())); blobWriter = blobServer; blobCache = blobServer; seenHashes.clear(); // verify that we do not upload the same content more than once doAnswer( invocation -> { PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod(); assertTrue(seenHashes.add(key.getHash())); return key; } ).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any()); blobServer.start(); }
Example #13
Source File: TaskExecutorTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setup() throws IOException { rpc = new TestingRpcService(); dummyBlobCacheService = new BlobCacheService( new Configuration(), new VoidBlobStore(), null); configuration = new Configuration(); unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation(); jobId = new JobID(); testingFatalErrorHandler = new TestingFatalErrorHandler(); haServices = new TestingHighAvailabilityServices(); resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(); jobManagerLeaderRetriever = new SettableLeaderRetrievalService(); haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever); haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever); nettyShuffleEnvironment = new NettyShuffleEnvironmentBuilder().build(); }
Example #14
Source File: TaskExecutorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Before public void setup() throws IOException { rpc = new TestingRpcService(); timerService = new TimerService<>(TestingUtils.defaultExecutor(), timeout.toMilliseconds()); dummyBlobCacheService = new BlobCacheService( new Configuration(), new VoidBlobStore(), null); configuration = new Configuration(); taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration); taskManagerLocation = new LocalTaskManagerLocation(); jobId = new JobID(); testingFatalErrorHandler = new TestingFatalErrorHandler(); haServices = new TestingHighAvailabilityServices(); resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(); jobManagerLeaderRetriever = new SettableLeaderRetrievalService(); haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever); haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever); }
Example #15
Source File: MiniDispatcherTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@BeforeClass public static void setupClass() throws IOException { jobGraph = new JobGraph(); archivedExecutionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobGraph.getJobID()) .setState(JobStatus.FINISHED) .build(); rpcService = new TestingRpcService(); configuration = new Configuration(); configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(configuration, new VoidBlobStore()); }
Example #16
Source File: ExecutionGraphDeploymentWithBlobServerTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void setupBlobServer() throws IOException { Configuration config = new Configuration(); // always offload the serialized job and task information config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0); blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore())); blobWriter = blobServer; blobCache = blobServer; seenHashes.clear(); // verify that we do not upload the same content more than once doAnswer( invocation -> { PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod(); assertTrue(seenHashes.add(key.getHash())); return key; } ).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any()); blobServer.start(); }
Example #17
Source File: DispatcherHATest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Nonnull private HATestingDispatcher createDispatcher( HighAvailabilityServices highAvailabilityServices, @Nonnull Queue<DispatcherId> fencingTokens, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception { final Configuration configuration = new Configuration(); TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway(); return new HATestingDispatcher( rpcService, UUID.randomUUID().toString(), configuration, highAvailabilityServices, () -> CompletableFuture.completedFuture(resourceManagerGateway), new BlobServer(configuration, new VoidBlobStore()), new HeartbeatServices(1000L, 1000L), UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), null, new MemoryArchivedExecutionGraphStore(), jobManagerRunnerFactory, testingFatalErrorHandler, fencingTokens); }
Example #18
Source File: ClientUtilsTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws IOException { Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); }
Example #19
Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws IOException, HandlerRequestException { final Configuration configuration = new Configuration(); configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(configuration, new VoidBlobStore()); handlerRequest = new HandlerRequest<>( EmptyRequestBody.getInstance(), new TaskManagerFileMessageParameters(), Collections.singletonMap(TaskManagerIdPathParameter.KEY, EXPECTED_TASK_MANAGER_ID.getResourceIdString()), Collections.emptyMap()); }
Example #20
Source File: JobSubmitHandlerTest.java From flink with Apache License 2.0 | 5 votes |
@Before public void setup() throws IOException { Configuration config = new Configuration(configuration); config.setString(BlobServerOptions.STORAGE_DIRECTORY, TEMPORARY_FOLDER.newFolder().getAbsolutePath()); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); }
Example #21
Source File: ExecutionGraphDeploymentWithBlobCacheTest.java From flink with Apache License 2.0 | 5 votes |
@Before @Override public void setupBlobServer() throws IOException { Configuration config = new Configuration(); // always offload the serialized job and task information config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); blobWriter = blobServer; InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort()); blobCache = new PermanentBlobCache(config, new VoidBlobStore(), serverAddress); }
Example #22
Source File: ZooKeeperRegistryTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning() */ @Test public void testZooKeeperRegistry() throws Exception { Configuration configuration = new Configuration(); configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString()); configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); final HighAvailabilityServices zkHaService = new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(configuration), Executors.directExecutor(), configuration, new VoidBlobStore()); final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry(); try { JobID jobID = JobID.generate(); assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID)); zkRegistry.setJobRunning(jobID); assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID)); zkRegistry.setJobFinished(jobID); assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID)); zkRegistry.clearJob(jobID); assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID)); } finally { zkHaService.close(); } }
Example #23
Source File: BlobServerResource.java From flink with Apache License 2.0 | 5 votes |
protected void before() throws Throwable { temporaryFolder.create(); Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); }
Example #24
Source File: BlobServerResource.java From flink with Apache License 2.0 | 5 votes |
protected void before() throws Throwable { temporaryFolder.create(); Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); }
Example #25
Source File: ZooKeeperRegistryTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning() */ @Test public void testZooKeeperRegistry() throws Exception { Configuration configuration = new Configuration(); configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString()); configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); final HighAvailabilityServices zkHaService = new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(configuration), Executors.directExecutor(), configuration, new VoidBlobStore()); final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry(); try { JobID jobID = JobID.generate(); assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID)); zkRegistry.setJobRunning(jobID); assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID)); zkRegistry.setJobFinished(jobID); assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID)); zkRegistry.clearJob(jobID); assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID)); } finally { zkHaService.close(); } }
Example #26
Source File: ExecutionGraphDeploymentWithBlobCacheTest.java From flink with Apache License 2.0 | 5 votes |
@Before @Override public void setupBlobServer() throws IOException { Configuration config = new Configuration(); // always offload the serialized job and task information config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); blobWriter = blobServer; InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort()); blobCache = new PermanentBlobCache(config, new VoidBlobStore(), serverAddress); }
Example #27
Source File: JobSubmitHandlerTest.java From flink with Apache License 2.0 | 5 votes |
@Before public void setup() throws IOException { Configuration config = new Configuration(configuration); config.setString(BlobServerOptions.STORAGE_DIRECTORY, TEMPORARY_FOLDER.newFolder().getAbsolutePath()); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); }
Example #28
Source File: ClientUtilsTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws IOException { Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); }
Example #29
Source File: ZooKeeperHADispatcherTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@BeforeClass public static void setupClass() throws IOException { configuration = new Configuration(); configuration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, TEMPORARY_FOLDER.newFolder().getAbsolutePath()); rpcService = new TestingRpcService(); blobServer = new BlobServer(configuration, new VoidBlobStore()); }
Example #30
Source File: DispatcherTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { final JobVertex testVertex = new JobVertex("testVertex"); testVertex.setInvokableClass(NoOpInvokable.class); jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex); jobGraph.setAllowQueuedScheduling(true); fatalErrorHandler = new TestingFatalErrorHandler(); heartbeatServices = new HeartbeatServices(1000L, 10000L); submittedJobGraphStore = new FaultySubmittedJobGraphStore(); dispatcherLeaderElectionService = new TestingLeaderElectionService(); jobMasterLeaderElectionService = new TestingLeaderElectionService(); haServices = new TestingHighAvailabilityServices(); haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService); haServices.setSubmittedJobGraphStore(submittedJobGraphStore); haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService); haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory()); haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService()); runningJobsRegistry = haServices.getRunningJobsRegistry(); configuration = new Configuration(); configuration.setString( BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); createdJobManagerRunnerLatch = new CountDownLatch(2); blobServer = new BlobServer(configuration, new VoidBlobStore()); }