Java Code Examples for org.apache.flink.api.common.time.Time#hours()

/**
 * Tests that we can cache AccessExecutionGraphs over multiple accesses.
 */
@Test
public void testExecutionGraphCaching() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	}
}
 

/**
 * Tests that we can cache AccessExecutionGraphs over multiple accesses.
 */
@Test
public void testExecutionGraphCaching() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	}
}
 

/**
 * Tests that we can cache AccessExecutionGraphs over multiple accesses.
 */
@Test
public void testExecutionGraphCaching() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	}
}
 

private FileArchivedExecutionGraphStore createDefaultExecutionGraphStore(File storageDirectory) throws IOException {
	return new FileArchivedExecutionGraphStore(
		storageDirectory,
		Time.hours(1L),
		10000L,
		TestingUtils.defaultScheduledExecutor(),
		Ticker.systemTicker());
}
 

/**
 * Tests that a failure in requesting an AccessExecutionGraph from the gateway, will not create
 * a cache entry --> another cache request will trigger a new gateway request.
 */
@Test
public void testImmediateCacheInvalidationAfterFailure() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	// let's first answer with a JobNotFoundException and then only with the correct result
	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(
		expectedJobId,
		FutureUtils.completedExceptionally(new FlinkJobNotFoundException(expectedJobId)),
		CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		try {
			executionGraphFuture.get();

			fail("The execution graph future should have been completed exceptionally.");
		} catch (ExecutionException ee) {
			assertTrue(ee.getCause() instanceof FlinkException);
		}

		CompletableFuture<AccessExecutionGraph> executionGraphFuture2 = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, executionGraphFuture2.get());
	}
}
 

/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 

private FileArchivedExecutionGraphStore createDefaultExecutionGraphStore(File storageDirectory) throws IOException {
	return new FileArchivedExecutionGraphStore(
		storageDirectory,
		Time.hours(1L),
		10000L,
		TestingUtils.defaultScheduledExecutor(),
		Ticker.systemTicker());
}
 

/**
 * Tests that a failure in requesting an AccessExecutionGraph from the gateway, will not create
 * a cache entry --> another cache request will trigger a new gateway request.
 */
@Test
public void testImmediateCacheInvalidationAfterFailure() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	// let's first answer with a JobNotFoundException and then only with the correct result
	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(
		expectedJobId,
		FutureUtils.completedExceptionally(new FlinkJobNotFoundException(expectedJobId)),
		CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		try {
			executionGraphFuture.get();

			fail("The execution graph future should have been completed exceptionally.");
		} catch (ExecutionException ee) {
			assertTrue(ee.getCause() instanceof FlinkException);
		}

		CompletableFuture<AccessExecutionGraph> executionGraphFuture2 = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, executionGraphFuture2.get());
	}
}
 

/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 

private FileArchivedExecutionGraphStore createDefaultExecutionGraphStore(File storageDirectory) throws IOException {
	return new FileArchivedExecutionGraphStore(
		storageDirectory,
		Time.hours(1L),
		Integer.MAX_VALUE,
		10000L,
		TestingUtils.defaultScheduledExecutor(),
		Ticker.systemTicker());
}
 

/**
 * Tests that a failure in requesting an AccessExecutionGraph from the gateway, will not create
 * a cache entry --> another cache request will trigger a new gateway request.
 */
@Test
public void testImmediateCacheInvalidationAfterFailure() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	// let's first answer with a JobNotFoundException and then only with the correct result
	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(
		expectedJobId,
		FutureUtils.completedExceptionally(new FlinkJobNotFoundException(expectedJobId)),
		CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		try {
			executionGraphFuture.get();

			fail("The execution graph future should have been completed exceptionally.");
		} catch (ExecutionException ee) {
			assertTrue(ee.getCause() instanceof FlinkException);
		}

		CompletableFuture<AccessExecutionGraph> executionGraphFuture2 = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, executionGraphFuture2.get());
	}
}
 

/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 

@Test
public void testSetTtlTimeProvider() throws Exception {
	AbstractStreamOperator<Integer> operator = new AbstractStreamOperator<Integer>() {};
	try (AbstractStreamOperatorTestHarness<Integer> result = new AbstractStreamOperatorTestHarness<>(
			operator,
			1,
			1,
			0)) {

		result.config.setStateKeySerializer(IntSerializer.INSTANCE);

		Time timeToLive = Time.hours(1);
		result.initializeState(new OperatorSubtaskState());
		result.open();

		ValueStateDescriptor<Integer> stateDescriptor = new ValueStateDescriptor<>("test", IntSerializer.INSTANCE);
		stateDescriptor.enableTimeToLive(StateTtlConfig.newBuilder(timeToLive).build());
		KeyedStateBackend<Integer> keyedStateBackend = operator.getKeyedStateBackend();
		ValueState<Integer> state = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		int expectedValue = 42;
		keyedStateBackend.setCurrentKey(1);
		result.setStateTtlProcessingTime(0L);
		state.update(expectedValue);
		Assert.assertEquals(expectedValue, (int) state.value());
		result.setStateTtlProcessingTime(timeToLive.toMilliseconds() + 1);
		Assert.assertNull(state.value());
	}
}