org.apache.flink.api.common.state.StateTtlConfig Java Examples

@Test
public void testExactExpirationOnRead() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnReadAndWrite, StateTtlConfig.StateVisibility.NeverReturnExpired);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 50;
	assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 120;
	assertEquals("Unexpired state should be available after read", ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 250;
	assertEquals(EXPIRED_UNAVAIL, ctx().emptyValue, ctx().get());
	assertEquals("Original state should be cleared on access", ctx().emptyValue, ctx().getOriginal());
}
 

@Test
public void testRelaxedExpirationOnRead() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnReadAndWrite, StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 50;
	assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 170;
	assertEquals(EXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());
	assertEquals("Expired state should be cleared on access", ctx().emptyValue, ctx().get());
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	setBackendWithCustomTTLTimeProvider(env);

	TtlTestConfig config = TtlTestConfig.fromArgs(pt);
	StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(config.ttl)
		.cleanupFullSnapshot()
		.cleanupInBackground()
		.build();

	env
		.addSource(new TtlStateUpdateSource(config.keySpace, config.sleepAfterElements, config.sleepTime))
		.name("TtlStateUpdateSource")
		.keyBy(TtlStateUpdate::getKey)
		.flatMap(new TtlVerifyUpdateFunction(ttlConfig, config.reportStatAfterUpdatesNum))
		.name("TtlVerifyUpdateFunction")
		.addSink(new PrintSinkFunction<>())
		.name("PrintFailedVerifications");

	env.execute("State TTL test job");
}
 

@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 

@Test
public void testRelaxedExpirationOnRead() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnReadAndWrite, StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 50;
	assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 170;
	assertEquals(EXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());
	assertEquals("Expired state should be cleared on access", ctx().emptyValue, ctx().get());
}
 

@Test
public void testExactExpirationOnRead() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnReadAndWrite, StateTtlConfig.StateVisibility.NeverReturnExpired);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 50;
	assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 120;
	assertEquals("Unexpired state should be available after read", ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 250;
	assertEquals(EXPIRED_UNAVAIL, ctx().emptyValue, ctx().get());
	assertEquals("Original state should be cleared on access", ctx().emptyValue, ctx().getOriginal());
}
 

private static StateTtlConfig from(Expiration expiration) {
  final long millis = expiration.duration().toMillis();
  Builder builder = StateTtlConfig.newBuilder(Time.milliseconds(millis));
  builder.setTtlTimeCharacteristic(TtlTimeCharacteristic.ProcessingTime);
  builder.setStateVisibility(StateVisibility.NeverReturnExpired);
  switch (expiration.mode()) {
    case AFTER_WRITE:
      {
        builder.setUpdateType(UpdateType.OnCreateAndWrite);
        break;
      }
    case AFTER_READ_OR_WRITE:
      {
        builder.setUpdateType(UpdateType.OnReadAndWrite);
        break;
      }
    default:
      throw new IllegalArgumentException("Unknown expiration mode " + expiration.mode());
  }
  return builder.build();
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    ValueStateDescriptor<Boolean> keyedStateDuplicated =
            new ValueStateDescriptor<>("KeyedStateDeduplication",
                    TypeInformation.of(new TypeHint<Boolean>() {}));
    // 状态 TTL 相关配置，过期时间设定为 36 小时
    StateTtlConfig ttlConfig = StateTtlConfig
            .newBuilder(Time.hours(36))
            .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
            .setStateVisibility(
                    StateTtlConfig.StateVisibility.NeverReturnExpired)
            .cleanupInRocksdbCompactFilter(50000000L)
            .build();
    // 开启 TTL
    keyedStateDuplicated.enableTimeToLive(ttlConfig);
    // 从状态后端恢复状态
    isExist = getRuntimeContext().getState(keyedStateDuplicated);
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	setBackendWithCustomTTLTimeProvider(env);

	TtlTestConfig config = TtlTestConfig.fromArgs(pt);
	StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(config.ttl)
		.cleanupFullSnapshot()
		.build();

	env
		.addSource(new TtlStateUpdateSource(config.keySpace, config.sleepAfterElements, config.sleepTime))
		.name("TtlStateUpdateSource")
		.keyBy(TtlStateUpdate::getKey)
		.flatMap(new TtlVerifyUpdateFunction(ttlConfig, config.reportStatAfterUpdatesNum))
		.name("TtlVerifyUpdateFunction")
		.addSink(new PrintSinkFunction<>())
		.name("PrintFailedVerifications");

	env.execute("State TTL test job");
}
 

private InputSideHasUniqueKey(
		RuntimeContext ctx,
		String stateName,
		RowDataTypeInfo recordType,
		RowDataTypeInfo uniqueKeyType,
		KeySelector<RowData, RowData> uniqueKeySelector,
		StateTtlConfig ttlConfig) {
	checkNotNull(uniqueKeyType);
	checkNotNull(uniqueKeySelector);
	TupleTypeInfo<Tuple2<RowData, Integer>> valueTypeInfo = new TupleTypeInfo<>(recordType, Types.INT);
	MapStateDescriptor<RowData, Tuple2<RowData, Integer>> recordStateDesc = new MapStateDescriptor<>(
		stateName,
		uniqueKeyType,
		valueTypeInfo);
	if (ttlConfig.isEnabled()) {
		recordStateDesc.enableTimeToLive(ttlConfig);
	}
	this.recordState = ctx.getMapState(recordStateDesc);
	this.uniqueKeySelector = uniqueKeySelector;
}
 

@Test
public void testRelaxedExpirationOnRead() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnReadAndWrite, StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 50;
	assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 170;
	assertEquals(EXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());
	assertEquals("Expired state should be cleared on access", ctx().emptyValue, ctx().get());
}
 

@Test
public void testExactExpirationOnRead() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnReadAndWrite, StateTtlConfig.StateVisibility.NeverReturnExpired);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 50;
	assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 120;
	assertEquals("Unexpired state should be available after read", ctx().getUpdateEmpty, ctx().get());

	takeAndRestoreSnapshot();

	timeProvider.time = 250;
	assertEquals(EXPIRED_UNAVAIL, ctx().emptyValue, ctx().get());
	assertTrue("Original state should be cleared on access", ctx().isOriginalEmptyValue());
}
 

@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	setBackendWithCustomTTLTimeProvider(env);

	TtlTestConfig config = TtlTestConfig.fromArgs(pt);
	StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(config.ttl)
		.cleanupFullSnapshot()
		.cleanupInBackground()
		.build();

	env
		.addSource(new TtlStateUpdateSource(config.keySpace, config.sleepAfterElements, config.sleepTime))
		.name("TtlStateUpdateSource")
		.keyBy(TtlStateUpdate::getKey)
		.flatMap(new TtlVerifyUpdateFunction(ttlConfig, config.reportStatAfterUpdatesNum))
		.name("TtlVerifyUpdateFunction")
		.addSink(new PrintSinkFunction<>())
		.name("PrintFailedVerifications");

	env.execute("State TTL test job");
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    ValueStateDescriptor<Boolean> keyedStateDuplicated =
            new ValueStateDescriptor<>("KeyedStateDeduplication",
                    TypeInformation.of(new TypeHint<Boolean>() {}));
    // 状态 TTL 相关配置，过期时间设定为 36 小时
    StateTtlConfig ttlConfig = StateTtlConfig
            .newBuilder(Time.hours(36))
            .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
            .setStateVisibility(
                    StateTtlConfig.StateVisibility.NeverReturnExpired)
            .cleanupInRocksdbCompactFilter(50000000L)
            .build();
    // 开启 TTL
    keyedStateDuplicated.enableTimeToLive(ttlConfig);
    // 从状态后端恢复状态
    isExist = getRuntimeContext().getState(keyedStateDuplicated);
}
 

/**
 * Creates a {@link OuterJoinRecordStateView} depends on {@link JoinInputSideSpec}.
 */
public static OuterJoinRecordStateView create(
		RuntimeContext ctx,
		String stateName,
		JoinInputSideSpec inputSideSpec,
		BaseRowTypeInfo recordType,
		long retentionTime,
		boolean stateCleaningEnabled) {
	StateTtlConfig ttlConfig = createTtlConfig(retentionTime, stateCleaningEnabled);
	if (inputSideSpec.hasUniqueKey()) {
		if (inputSideSpec.joinKeyContainsUniqueKey()) {
			return new OuterJoinRecordStateViews.JoinKeyContainsUniqueKey(ctx, stateName, recordType, ttlConfig);
		} else {
			return new OuterJoinRecordStateViews.InputSideHasUniqueKey(
				ctx,
				stateName,
				recordType,
				inputSideSpec.getUniqueKeyType(),
				inputSideSpec.getUniqueKeySelector(),
				ttlConfig);
		}
	} else {
		return new OuterJoinRecordStateViews.InputSideHasNoUniqueKey(ctx, stateName, recordType, ttlConfig);
	}
}
 

private InputSideHasUniqueKey(
		RuntimeContext ctx,
		String stateName,
		BaseRowTypeInfo recordType,
		BaseRowTypeInfo uniqueKeyType,
		KeySelector<BaseRow, BaseRow> uniqueKeySelector,
		StateTtlConfig ttlConfig) {
	checkNotNull(uniqueKeyType);
	checkNotNull(uniqueKeySelector);
	TupleTypeInfo<Tuple2<BaseRow, Integer>> valueTypeInfo = new TupleTypeInfo<>(recordType, Types.INT);
	MapStateDescriptor<BaseRow, Tuple2<BaseRow, Integer>> recordStateDesc = new MapStateDescriptor<>(
		stateName,
		uniqueKeyType,
		valueTypeInfo);
	if (!ttlConfig.equals(StateTtlConfig.DISABLED)) {
		recordStateDesc.enableTimeToLive(ttlConfig);
	}
	this.recordState = ctx.getMapState(recordStateDesc);
	this.uniqueKeySelector = uniqueKeySelector;
}
 

/**
 * Creates a {@link JoinRecordStateView} depends on {@link JoinInputSideSpec}.
 */
public static JoinRecordStateView create(
		RuntimeContext ctx,
		String stateName,
		JoinInputSideSpec inputSideSpec,
		RowDataTypeInfo recordType,
		long retentionTime) {
	StateTtlConfig ttlConfig = createTtlConfig(retentionTime);
	if (inputSideSpec.hasUniqueKey()) {
		if (inputSideSpec.joinKeyContainsUniqueKey()) {
			return new JoinKeyContainsUniqueKey(ctx, stateName, recordType, ttlConfig);
		} else {
			return new InputSideHasUniqueKey(
				ctx,
				stateName,
				recordType,
				inputSideSpec.getUniqueKeyType(),
				inputSideSpec.getUniqueKeySelector(),
				ttlConfig);
		}
	} else {
		return new InputSideHasNoUniqueKey(ctx, stateName, recordType, ttlConfig);
	}
}
 

/**
 * Creates a {@link JoinRecordStateView} depends on {@link JoinInputSideSpec}.
 */
public static JoinRecordStateView create(
		RuntimeContext ctx,
		String stateName,
		JoinInputSideSpec inputSideSpec,
		BaseRowTypeInfo recordType,
		long retentionTime,
		boolean stateCleaningEnabled) {
	StateTtlConfig ttlConfig = createTtlConfig(retentionTime, stateCleaningEnabled);
	if (inputSideSpec.hasUniqueKey()) {
		if (inputSideSpec.joinKeyContainsUniqueKey()) {
			return new JoinKeyContainsUniqueKey(ctx, stateName, recordType, ttlConfig);
		} else {
			return new InputSideHasUniqueKey(
				ctx,
				stateName,
				recordType,
				inputSideSpec.getUniqueKeyType(),
				inputSideSpec.getUniqueKeySelector(),
				ttlConfig);
		}
	} else {
		return new InputSideHasNoUniqueKey(ctx, stateName, recordType, ttlConfig);
	}
}
 

private InputSideHasUniqueKey(
		RuntimeContext ctx,
		String stateName,
		BaseRowTypeInfo recordType,
		BaseRowTypeInfo uniqueKeyType,
		KeySelector<BaseRow, BaseRow> uniqueKeySelector,
		StateTtlConfig ttlConfig) {
	checkNotNull(uniqueKeyType);
	checkNotNull(uniqueKeySelector);
	MapStateDescriptor<BaseRow, BaseRow> recordStateDesc = new MapStateDescriptor<>(
		stateName,
		uniqueKeyType,
		recordType);
	if (!ttlConfig.equals(StateTtlConfig.DISABLED)) {
		recordStateDesc.enableTimeToLive(ttlConfig);
	}
	this.recordState = ctx.getMapState(recordStateDesc);
	this.uniqueKeySelector = uniqueKeySelector;
}
 

@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 

@Test
public void testRelaxedExpirationOnWrite() throws Exception {
	initTest(StateTtlConfig.UpdateType.OnCreateAndWrite, StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp);

	timeProvider.time = 0;
	ctx().update(ctx().updateEmpty);

	takeAndRestoreSnapshot();

	timeProvider.time = 120;
	assertEquals(EXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());
	assertEquals("Original state should be cleared on access", ctx().emptyValue, ctx().getOriginal());
	assertEquals("Expired state should be cleared on access", ctx().emptyValue, ctx().get());
}
 

private InputSideHasNoUniqueKey(
		RuntimeContext ctx,
		String stateName,
		RowDataTypeInfo recordType,
		StateTtlConfig ttlConfig) {
	TupleTypeInfo<Tuple2<Integer, Integer>> tupleTypeInfo = new TupleTypeInfo<>(Types.INT, Types.INT);
	MapStateDescriptor<RowData, Tuple2<Integer, Integer>> recordStateDesc = new MapStateDescriptor<>(
		stateName,
		recordType,
		tupleTypeInfo);
	if (ttlConfig.isEnabled()) {
		recordStateDesc.enableTimeToLive(ttlConfig);
	}
	this.recordState = ctx.getMapState(recordStateDesc);
}
 

@SuppressWarnings("unchecked")
@Override
@Nonnull
public State createState(@Nonnull FunctionInitializationContext context, @Nonnull StateTtlConfig ttlConfig) {
	stateDesc.enableTimeToLive(ttlConfig);
	return createState(context);
}
 

static void configureStateTtl(StateDescriptor<?, ?> handle, Expiration expiration) {
  if (expiration.mode() == Mode.NONE) {
    return;
  }
  StateTtlConfig ttlConfig = from(expiration);
  handle.enableTimeToLive(ttlConfig);
}
 

AbstractTtlDecorator(
	T original,
	StateTtlConfig config,
	TtlTimeProvider timeProvider) {
	Preconditions.checkNotNull(original);
	Preconditions.checkNotNull(config);
	Preconditions.checkNotNull(timeProvider);
	this.original = original;
	this.config = config;
	this.timeProvider = timeProvider;
	this.updateTsOnRead = config.getUpdateType() == StateTtlConfig.UpdateType.OnReadAndWrite;
	this.returnExpired = config.getStateVisibility() == StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp;
	this.ttl = config.getTtl().toMilliseconds();
}
 

private Runnable registerTtlIncrementalCleanupCallback(InternalKvState<?, ?, ?> originalState) {
	StateTtlConfig.IncrementalCleanupStrategy config =
		ttlConfig.getCleanupStrategies().getIncrementalCleanupStrategy();
	boolean cleanupConfigured = config != null && incrementalCleanup != null;
	boolean isCleanupActive = cleanupConfigured &&
		isStateIteratorSupported(originalState, incrementalCleanup.getCleanupSize());
	Runnable callback = isCleanupActive ? incrementalCleanup::stateAccessed : () -> { };
	if (isCleanupActive && config.runCleanupForEveryRecord()) {
		stateBackend.registerKeySelectionListener(stub -> callback.run());
	}
	return callback;
}
 

static StateTtlConfig createTtlConfig(long retentionTime, boolean stateCleaningEnabled) {
	if (stateCleaningEnabled) {
		checkArgument(retentionTime > 0);
		return StateTtlConfig
			.newBuilder(Time.milliseconds(retentionTime))
			.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
			.setStateVisibility(StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp)
			.build();
	} else {
		return StateTtlConfig.DISABLED;
	}
}
 

TtlStateContext(
	T original,
	StateTtlConfig config,
	TtlTimeProvider timeProvider,
	TypeSerializer<SV> valueSerializer,
	Runnable accessCallback) {
	this.original = original;
	this.config = config;
	this.timeProvider = timeProvider;
	this.valueSerializer = valueSerializer;
	this.accessCallback = accessCallback;
}
 

private JoinKeyContainsUniqueKey(
		RuntimeContext ctx,
		String stateName,
		RowDataTypeInfo recordType,
		StateTtlConfig ttlConfig) {
	ValueStateDescriptor<RowData> recordStateDesc = new ValueStateDescriptor<>(
		stateName,
		recordType);
	if (ttlConfig.isEnabled()) {
		recordStateDesc.enableTimeToLive(ttlConfig);
	}
	this.recordState = ctx.getState(recordStateDesc);
	// the result records always not more than 1
	this.reusedList = new ArrayList<>(1);
}