Java Code Examples for org.apache.flink.api.common.time.Time#minutes()

MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new TestingHeartbeatServices();
	metricRegistry = NoOpMetricRegistry.INSTANCE;
	slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()))
		.setTaskManagerRequestTimeout(Time.seconds(10))
		.setSlotRequestTimeout(Time.seconds(10))
		.setTaskManagerTimeout(Time.minutes(1))
		.build();
	jobLeaderIdService = new JobLeaderIdService(
			highAvailabilityServices,
			rpcService.getScheduledExecutor(),
			Time.minutes(5L));
}
 

@Test
public void testRestore() throws Exception {
    ReaderGroup readerGroup = mock(ReaderGroup.class);

    Checkpoint checkpoint = mock(Checkpoint.class);
    CheckpointImpl checkpointImpl = mock(CheckpointImpl.class);

    when(checkpoint.asImpl()).thenReturn(checkpointImpl);
    when(checkpointImpl.getPositions()).thenReturn(ImmutableMap.<Stream, StreamCut>builder()
            .put(Stream.of(SCOPE, "s1"), getStreamCut("s1"))
            .put(Stream.of(SCOPE, "s2"), getStreamCut("s2")).build());

    ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
            .disableAutomaticCheckpoints()
            .startFromCheckpoint(checkpoint)
            .build();

    TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1),
            readerGroupConfig);

    hook.restoreCheckpoint(1L, checkpoint);

    verify(readerGroup).resetReaderGroup(readerGroupConfig);
}
 

@Test
public void testTriggerCheckpointTimeout() throws Exception {
    ReaderGroup readerGroup = mock(ReaderGroup.class);
    ReaderGroupConfig readerGroupConfig = mock(ReaderGroupConfig.class);
    CompletableFuture<Checkpoint> checkpointPromise = new CompletableFuture<>();
    when(readerGroup.initiateCheckpoint(anyString(), any())).thenReturn(checkpointPromise);
    TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1), readerGroupConfig);

    CompletableFuture<Checkpoint> checkpointFuture = hook.triggerCheckpoint(1L, 1L, Executors.directExecutor());
    assertNotNull(checkpointFuture);
    verify(readerGroup).initiateCheckpoint(anyString(), any());

    // invoke the timeout callback
    hook.invokeScheduledCallables();
    assertTrue(checkpointFuture.isCancelled());
}
 

@Test
public void testTriggerCheckpoint() throws Exception {
    ReaderGroup readerGroup = mock(ReaderGroup.class);
    ReaderGroupConfig readerGroupConfig = mock(ReaderGroupConfig.class);
    CompletableFuture<Checkpoint> checkpointPromise = new CompletableFuture<>();
    when(readerGroup.initiateCheckpoint(anyString(), any())).thenReturn(checkpointPromise);
    TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1), readerGroupConfig);

    CompletableFuture<Checkpoint> checkpointFuture = hook.triggerCheckpoint(1L, 1L, Executors.directExecutor());
    assertNotNull(checkpointFuture);
    verify(readerGroup).initiateCheckpoint(anyString(), any());

    // complete the checkpoint promise
    Checkpoint expectedCheckpoint = mock(Checkpoint.class);
    checkpointPromise.complete(expectedCheckpoint);
    assertTrue(checkpointFuture.isDone());
    assertSame(expectedCheckpoint, checkpointFuture.get());
}
 

MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new HeartbeatServices(Integer.MAX_VALUE, Integer.MAX_VALUE);
	slotManager = mock(SlotManager.class);
	slotManagerStarted = new CompletableFuture<>();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			rmActions = invocation.getArgument(2);
			slotManagerStarted.complete(true);
			return null;
		}
	}).when(slotManager).start(any(ResourceManagerId.class), any(Executor.class), any(ResourceActions.class));

	when(slotManager.registerSlotRequest(any(SlotRequest.class))).thenReturn(true);
}
 

MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new HeartbeatServices(5L, 5L);
	metricRegistry = mock(MetricRegistryImpl.class);
	slotManager = mock(SlotManager.class);
	slotManagerStarted = new CompletableFuture<>();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			rmActions = invocation.getArgument(2);
			slotManagerStarted.complete(true);
			return null;
		}
	}).when(slotManager).start(any(ResourceManagerId.class), any(Executor.class), any(ResourceActions.class));

	when(slotManager.registerSlotRequest(any(SlotRequest.class))).thenReturn(true);
}
 

@Test
public void testRetractStreamDoesNotOverwriteTableConfig() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStreamSource<Integer> elements = env.fromElements(1, 2, 3);

	StreamTableEnvironmentImpl tEnv = getStreamTableEnvironment(env, elements);

	Time minRetention = Time.minutes(1);
	Time maxRetention = Time.minutes(10);
	tEnv.getConfig().setIdleStateRetentionTime(minRetention, maxRetention);
	Table table = tEnv.fromDataStream(elements);
	tEnv.toRetractStream(table, Row.class);

	assertThat(
		tEnv.getConfig().getMinIdleStateRetentionTime(),
		equalTo(minRetention.toMilliseconds()));
	assertThat(
		tEnv.getConfig().getMaxIdleStateRetentionTime(),
		equalTo(maxRetention.toMilliseconds()));
}
 

@Test
public void testAppendStreamDoesNotOverwriteTableConfig() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStreamSource<Integer> elements = env.fromElements(1, 2, 3);

	StreamTableEnvironmentImpl tEnv = getStreamTableEnvironment(env, elements);

	Time minRetention = Time.minutes(1);
	Time maxRetention = Time.minutes(10);
	tEnv.getConfig().setIdleStateRetentionTime(minRetention, maxRetention);
	Table table = tEnv.fromDataStream(elements);
	tEnv.toAppendStream(table, Row.class);

	assertThat(
		tEnv.getConfig().getMinIdleStateRetentionTime(),
		equalTo(minRetention.toMilliseconds()));
	assertThat(
		tEnv.getConfig().getMaxIdleStateRetentionTime(),
		equalTo(maxRetention.toMilliseconds()));
}
 

MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new HeartbeatServices(5L, 5L);
	metricRegistry = mock(MetricRegistryImpl.class);
	slotManager = mock(SlotManager.class);
	slotManagerStarted = new CompletableFuture<>();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			rmActions = invocation.getArgument(2);
			slotManagerStarted.complete(true);
			return null;
		}
	}).when(slotManager).start(any(ResourceManagerId.class), any(Executor.class), any(ResourceActions.class));

	when(slotManager.registerSlotRequest(any(SlotRequest.class))).thenReturn(true);
}
 

private ResourceManager<?> createAndStartResourceManager() throws Exception {
	ResourceID rmResourceId = ResourceID.generate();

	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, 1000L);

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		haServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	ResourceManager<?> resourceManager = new StandaloneResourceManager(
		rpcService,
		rmResourceId,
		haServices,
		heartbeatServices,
		slotManager,
		NoOpResourceManagerPartitionTracker::get,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
		Time.minutes(5L),
		RpcUtils.INF_TIMEOUT);

	resourceManager.start();

	return resourceManager;
}
 

private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpMetricRegistry.INSTANCE,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
			Time.minutes(5L));

	resourceManager.start();

	return resourceManager;
}
 

public MockResourceManagerRuntimeServices(RpcService rpcService, Time timeout, SlotManager slotManager) {
	this.rpcService = checkNotNull(rpcService);
	this.timeout = checkNotNull(timeout);
	this.slotManager = slotManager;
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new TestingHeartbeatServices();
	metricRegistry = NoOpMetricRegistry.INSTANCE;
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));
}
 

/**
 * Tests that the lazy scheduling flag is correctly forwarded to the produced partition descriptors.
 */
@Test
public void testTddProducedPartitionsLazyScheduling() throws Exception {
	ExecutionJobVertex jobVertex = getExecutionVertex(new JobVertexID(), new DirectScheduledExecutorService());

	IntermediateResult result =
			new IntermediateResult(new IntermediateDataSetID(), jobVertex, 1, ResultPartitionType.PIPELINED);

	ExecutionVertex vertex =
			new ExecutionVertex(jobVertex, 0, new IntermediateResult[]{result}, Time.minutes(1));

	ExecutionEdge mockEdge = createMockExecutionEdge(1);

	result.getPartitions()[0].addConsumerGroup();
	result.getPartitions()[0].addConsumer(mockEdge, 0);

	SlotContext slotContext = mock(SlotContext.class);
	when(slotContext.getAllocationId()).thenReturn(new AllocationID());

	LogicalSlot slot = mock(LogicalSlot.class);
	when(slot.getAllocationId()).thenReturn(new AllocationID());

	for (ScheduleMode mode : ScheduleMode.values()) {
		vertex.getExecutionGraph().setScheduleMode(mode);

		TaskDeploymentDescriptor tdd = vertex.createDeploymentDescriptor(new ExecutionAttemptID(), slot, null, 1);

		Collection<ResultPartitionDeploymentDescriptor> producedPartitions = tdd.getProducedPartitions();

		assertEquals(1, producedPartitions.size());
		ResultPartitionDeploymentDescriptor desc = producedPartitions.iterator().next();
		assertEquals(mode.allowLazyDeployment(), desc.sendScheduleOrUpdateConsumersMessage());
	}
}
 

/**
 * Tests that the lazy scheduling flag is correctly forwarded to the produced partition descriptors.
 */
@Test
public void testTddProducedPartitionsLazyScheduling() throws Exception {
	for (ScheduleMode scheduleMode: ScheduleMode.values()) {
		ExecutionJobVertex jobVertex = getExecutionVertex(
			new JobVertexID(),
			new DirectScheduledExecutorService(),
			scheduleMode);

		IntermediateResult result =
			new IntermediateResult(new IntermediateDataSetID(), jobVertex, 1, ResultPartitionType.PIPELINED);

		ExecutionAttemptID attemptID = new ExecutionAttemptID();
		ExecutionVertex vertex =
			new ExecutionVertex(jobVertex, 0, new IntermediateResult[]{result}, Time.minutes(1));
		TaskDeploymentDescriptorFactory tddFactory =
			TaskDeploymentDescriptorFactory.fromExecutionVertex(vertex, 1);

		ExecutionEdge mockEdge = createMockExecutionEdge(1);

		result.getPartitions()[0].addConsumerGroup();
		result.getPartitions()[0].addConsumer(mockEdge, 0);

		TaskManagerLocation location =
			new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 1);

		TaskDeploymentDescriptor tdd = tddFactory.createDeploymentDescriptor(
			new AllocationID(),
			0,
			null,
			Execution.registerProducedPartitions(vertex, location, attemptID).get().values());

		Collection<ResultPartitionDeploymentDescriptor> producedPartitions = tdd.getProducedPartitions();

		assertEquals(1, producedPartitions.size());
		ResultPartitionDeploymentDescriptor desc = producedPartitions.iterator().next();
		assertEquals(scheduleMode.allowLazyDeployment(), desc.sendScheduleOrUpdateConsumersMessage());
	}
}
 

public MockResourceManagerRuntimeServices(RpcService rpcService, Time timeout, SlotManager slotManager) {
	this.rpcService = checkNotNull(rpcService);
	this.timeout = checkNotNull(timeout);
	this.slotManager = slotManager;
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new TestingHeartbeatServices();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));
}
 

@Test
public void testConstructor() throws Exception {
    ReaderGroup readerGroup = mock(ReaderGroup.class);
    ReaderGroupConfig readerGroupConfig = mock(ReaderGroupConfig.class);
    TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1), readerGroupConfig);
    assertEquals(HOOK_UID, hook.getIdentifier());
    assertTrue(hook.createCheckpointDataSerializer() instanceof CheckpointSerializer);
}
 

private ResourceManager<?> createAndStartResourceManager() throws Exception {
	ResourceID rmResourceId = ResourceID.generate();

	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, 1000L);

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		haServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	ResourceManager<?> resourceManager = new StandaloneResourceManager(
		rpcService,
		ResourceManager.RESOURCE_MANAGER_NAME,
		rmResourceId,
		haServices,
		heartbeatServices,
		slotManager,
		NoOpMetricRegistry.INSTANCE,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	return resourceManager;
}
 

private static IntermediateResult createResult(
		ResultPartitionType resultPartitionType,
		int producerCount) throws Exception {

	ExecutionJobVertex jobVertex = getExecutionJobVertex(new JobVertexID(), new DirectScheduledExecutorService());
	IntermediateResult result =
			new IntermediateResult(new IntermediateDataSetID(), jobVertex, producerCount, resultPartitionType);
	for (int i = 0; i < producerCount; i++) {
		// Generate result partition in the result
		new ExecutionVertex(jobVertex, i, new IntermediateResult[]{result}, Time.minutes(1));
	}

	return result;
}
 

private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpResourceManagerPartitionTracker::get,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
			Time.minutes(5L),
			RpcUtils.INF_TIMEOUT);

	resourceManager.start();

	return resourceManager;
}
 

@Test
public void testClose() {
    ReaderGroup readerGroup = mock(ReaderGroup.class);
    ReaderGroupConfig readerGroupConfig = mock(ReaderGroupConfig.class);
    TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1), readerGroupConfig);
    hook.close();
    verify(readerGroup).close();
    assertNull(hook.getScheduledExecutorService());
}